From 0a2ee2fd53b4759e9041af8620717127a7e81fc3 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 12 Aug 2014 17:46:31 +0000 Subject: [PATCH 01/67] HADOOP-10820. Throw an exception in GenericOptionsParser when passed an empty Path. Contributed by Alex Holmes and Zhihai Xu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617542 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/util/GenericOptionsParser.java | 6 ++ .../hadoop/util/TestGenericOptionsParser.java | 66 +++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c29934be445..08c981e1e58 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -502,6 +502,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10835. Implement HTTP proxyuser support in HTTP authentication client/server libraries. (tucu) + HADOOP-10820. Throw an exception in GenericOptionsParser when passed + an empty Path. (Alex Holmes and Zhihai Xu via wang) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java index cb6f91c7c67..18acbf109a2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java @@ -378,9 +378,15 @@ public class GenericOptionsParser { if (files == null) return null; String[] fileArr = files.split(","); + if (fileArr.length == 0) { + throw new IllegalArgumentException("File name can't be empty string"); + } String[] finalArr = new String[fileArr.length]; for (int i =0; i < fileArr.length; i++) { String tmp = fileArr[i]; + if (tmp.isEmpty()) { + throw new IllegalArgumentException("File name can't be empty string"); + } String finalPath; URI pathURI; try { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java index 48a419b3a52..779318acc8e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java @@ -21,11 +21,14 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import junit.framework.TestCase; +import org.apache.commons.math3.util.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -34,12 +37,14 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.junit.Assert; import com.google.common.collect.Maps; +import static org.junit.Assert.fail; public class TestGenericOptionsParser extends TestCase { File testDir; @@ -92,6 +97,67 @@ public class TestGenericOptionsParser extends TestCase { assertNull("files is not null", files); } + /** + * Test the case where the libjars, files and archives arguments + * contains an empty token, which should create an IllegalArgumentException. + */ + public void testEmptyFilenames() throws Exception { + List> argsAndConfNames = new ArrayList>(); + argsAndConfNames.add(new Pair("-libjars", "tmpjars")); + argsAndConfNames.add(new Pair("-files", "tmpfiles")); + argsAndConfNames.add(new Pair("-archives", "tmparchives")); + for (Pair argAndConfName : argsAndConfNames) { + String arg = argAndConfName.getFirst(); + String configName = argAndConfName.getSecond(); + + File tmpFileOne = new File(testDir, "tmpfile1"); + Path tmpPathOne = new Path(tmpFileOne.toString()); + File tmpFileTwo = new File(testDir, "tmpfile2"); + Path tmpPathTwo = new Path(tmpFileTwo.toString()); + localFs.create(tmpPathOne); + localFs.create(tmpPathTwo); + String[] args = new String[2]; + args[0] = arg; + // create an empty path in between two valid files, + // which prior to HADOOP-10820 used to result in the + // working directory being added to "tmpjars" (or equivalent) + args[1] = String.format("%s,,%s", + tmpFileOne.toURI().toString(), tmpFileTwo.toURI().toString()); + try { + new GenericOptionsParser(conf, args); + fail("Expected exception for empty filename"); + } catch (IllegalArgumentException e) { + // expect to receive an IllegalArgumentException + GenericTestUtils.assertExceptionContains("File name can't be" + + " empty string", e); + } + + // test zero file list length - it should create an exception + args[1] = ",,"; + try { + new GenericOptionsParser(conf, args); + fail("Expected exception for zero file list length"); + } catch (IllegalArgumentException e) { + // expect to receive an IllegalArgumentException + GenericTestUtils.assertExceptionContains("File name can't be" + + " empty string", e); + } + + // test filename with space character + // it should create exception from parser in URI class + // due to URI syntax error + args[1] = String.format("%s, ,%s", + tmpFileOne.toURI().toString(), tmpFileTwo.toURI().toString()); + try { + new GenericOptionsParser(conf, args); + fail("Expected exception for filename with space character"); + } catch (IllegalArgumentException e) { + // expect to receive an IllegalArgumentException + GenericTestUtils.assertExceptionContains("URISyntaxException", e); + } + } + } + /** * Test that options passed to the constructor are used. */ From 0c9b8f2d7ffa3e7c36bc317ff3facb992f7a154c Mon Sep 17 00:00:00 2001 From: Jian He Date: Tue, 12 Aug 2014 18:29:42 +0000 Subject: [PATCH 02/67] YARN-2373. Changed WebAppUtils to use Configuration#getPassword for accessing SSL passwords. Contributed by Larry McCay git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617555 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/webapp/util/WebAppUtils.java | 51 +++++- .../yarn/webapp/util/TestWebAppUtils.java | 148 ++++++++++++++++++ 3 files changed, 197 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f583143f060..c50e9eae313 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -116,6 +116,9 @@ Release 2.6.0 - UNRELEASED YARN-2138. Cleaned up notifyDone* APIs in RMStateStore. (Varun Saxena via jianhe) + YARN-2373. Changed WebAppUtils to use Configuration#getPassword for + accessing SSL passwords. (Larry McCay via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 6cbe6f94d6f..a8f67ff569f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -19,12 +19,12 @@ package org.apache.hadoop.yarn.webapp.util; import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; +import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -40,6 +40,12 @@ import org.apache.hadoop.yarn.util.RMHAUtils; @Private @Evolving public class WebAppUtils { + public static final String WEB_APP_TRUSTSTORE_PASSWORD_KEY = + "ssl.server.truststore.password"; + public static final String WEB_APP_KEYSTORE_PASSWORD_KEY = + "ssl.server.keystore.password"; + public static final String WEB_APP_KEY_PASSWORD_KEY = + "ssl.server.keystore.keypassword"; public static final String HTTPS_PREFIX = "https://"; public static final String HTTP_PREFIX = "http://"; @@ -274,21 +280,56 @@ public class WebAppUtils { /** * Load the SSL keystore / truststore into the HttpServer builder. + * @param builder the HttpServer2.Builder to populate with ssl config */ public static HttpServer2.Builder loadSslConfiguration( HttpServer2.Builder builder) { - Configuration sslConf = new Configuration(false); + return loadSslConfiguration(builder, null); + } + + /** + * Load the SSL keystore / truststore into the HttpServer builder. + * @param builder the HttpServer2.Builder to populate with ssl config + * @param sslConf the Configuration instance to use during loading of SSL conf + */ + public static HttpServer2.Builder loadSslConfiguration( + HttpServer2.Builder builder, Configuration sslConf) { + if (sslConf == null) { + sslConf = new Configuration(false); + } boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT; sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT); return builder .needsClientAuth(needsClientAuth) - .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) + .keyPassword(getPassword(sslConf, WEB_APP_KEY_PASSWORD_KEY)) .keyStore(sslConf.get("ssl.server.keystore.location"), - sslConf.get("ssl.server.keystore.password"), + getPassword(sslConf, WEB_APP_KEYSTORE_PASSWORD_KEY), sslConf.get("ssl.server.keystore.type", "jks")) .trustStore(sslConf.get("ssl.server.truststore.location"), - sslConf.get("ssl.server.truststore.password"), + getPassword(sslConf, WEB_APP_TRUSTSTORE_PASSWORD_KEY), sslConf.get("ssl.server.truststore.type", "jks")); } + + /** + * Leverages the Configuration.getPassword method to attempt to get + * passwords from the CredentialProvider API before falling back to + * clear text in config - if falling back is allowed. + * @param conf Configuration instance + * @param alias name of the credential to retreive + * @return String credential value or null + */ + static String getPassword(Configuration conf, String alias) { + String password = null; + try { + char[] passchars = conf.getPassword(alias); + if (passchars != null) { + password = new String(passchars); + } + } + catch (IOException ioe) { + password = null; + } + return password; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java new file mode 100644 index 00000000000..18600fdea68 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java @@ -0,0 +1,148 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.webapp.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.http.HttpServer2.Builder; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.junit.Assert; +import org.junit.Test; + +public class TestWebAppUtils { + + @Test + public void testGetPassword() throws Exception { + Configuration conf = provisionCredentialsForSSL(); + + // use WebAppUtils as would be used by loadSslConfiguration + Assert.assertEquals("keypass", + WebAppUtils.getPassword(conf, WebAppUtils.WEB_APP_KEY_PASSWORD_KEY)); + Assert.assertEquals("storepass", + WebAppUtils.getPassword(conf, WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY)); + Assert.assertEquals("trustpass", + WebAppUtils.getPassword(conf, WebAppUtils.WEB_APP_TRUSTSTORE_PASSWORD_KEY)); + + // let's make sure that a password that doesn't exist returns null + Assert.assertEquals(null, WebAppUtils.getPassword(conf,"invalid-alias")); + } + + @Test + public void testLoadSslConfiguration() throws Exception { + Configuration conf = provisionCredentialsForSSL(); + TestBuilder builder = (TestBuilder) new TestBuilder(); + + builder = (TestBuilder) WebAppUtils.loadSslConfiguration( + builder, conf); + + String keypass = "keypass"; + String storepass = "storepass"; + String trustpass = "trustpass"; + + // make sure we get the right passwords in the builder + assertEquals(keypass, ((TestBuilder)builder).keypass); + assertEquals(storepass, ((TestBuilder)builder).keystorePassword); + assertEquals(trustpass, ((TestBuilder)builder).truststorePassword); + } + + protected Configuration provisionCredentialsForSSL() throws IOException, + Exception { + File testDir = new File(System.getProperty("test.build.data", + "target/test-dir")); + + Configuration conf = new Configuration(); + final String ourUrl = + JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; + + File file = new File(testDir, "test.jks"); + file.delete(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); + + CredentialProvider provider = + CredentialProviderFactory.getProviders(conf).get(0); + char[] keypass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; + char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'}; + char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'}; + + // ensure that we get nulls when the key isn't there + assertEquals(null, provider.getCredentialEntry( + WebAppUtils.WEB_APP_KEY_PASSWORD_KEY)); + assertEquals(null, provider.getCredentialEntry( + WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY)); + assertEquals(null, provider.getCredentialEntry( + WebAppUtils.WEB_APP_TRUSTSTORE_PASSWORD_KEY)); + + // create new aliases + try { + provider.createCredentialEntry( + WebAppUtils.WEB_APP_KEY_PASSWORD_KEY, keypass); + + provider.createCredentialEntry( + WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY, storepass); + + provider.createCredentialEntry( + WebAppUtils.WEB_APP_TRUSTSTORE_PASSWORD_KEY, trustpass); + + // write out so that it can be found in checks + provider.flush(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + // make sure we get back the right key directly from api + assertArrayEquals(keypass, provider.getCredentialEntry( + WebAppUtils.WEB_APP_KEY_PASSWORD_KEY).getCredential()); + assertArrayEquals(storepass, provider.getCredentialEntry( + WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY).getCredential()); + assertArrayEquals(trustpass, provider.getCredentialEntry( + WebAppUtils.WEB_APP_TRUSTSTORE_PASSWORD_KEY).getCredential()); + return conf; + } + + public class TestBuilder extends HttpServer2.Builder { + public String keypass; + public String keystorePassword; + public String truststorePassword; + + @Override + public Builder trustStore(String location, String password, String type) { + truststorePassword = password; + return super.trustStore(location, password, type); + } + + @Override + public Builder keyStore(String location, String password, String type) { + keystorePassword = password; + return super.keyStore(location, password, type); + } + + @Override + public Builder keyPassword(String password) { + keypass = password; + return super.keyPassword(password); + } + } +} From 7ad7ab1723e246c0959b5bdf28d1b0f7dcd8a9bc Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Tue, 12 Aug 2014 21:37:34 +0000 Subject: [PATCH 03/67] YARN-2317. Updated the document about how to write YARN applications. Contributed by Li Lu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617594 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../site/apt/WritingYarnApplications.apt.vm | 1317 ++++++++--------- 2 files changed, 644 insertions(+), 676 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c50e9eae313..af0f9b56e32 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -119,6 +119,9 @@ Release 2.6.0 - UNRELEASED YARN-2373. Changed WebAppUtils to use Configuration#getPassword for accessing SSL passwords. (Larry McCay via jianhe) + YARN-2317. Updated the document about how to write YARN applications. (Li Lu via + zjshen) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm index 12e4c3ff334..57a47fda59d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm @@ -11,8 +11,8 @@ ~~ limitations under the License. See accompanying LICENSE file. --- - Hadoop Map Reduce Next Generation-${project.version} - Writing YARN - Applications + Hadoop Map Reduce Next Generation-${project.version} - Writing YARN + Applications --- --- ${maven.build.timestamp} @@ -21,772 +21,737 @@ Hadoop MapReduce Next Generation - Writing YARN Applications %{toc|section=1|fromDepth=0} -* Purpose +* Purpose - This document describes, at a high-level, the way to implement new + This document describes, at a high-level, the way to implement new Applications for YARN. * Concepts and Flow - The general concept is that an 'Application Submission Client' submits an - 'Application' to the YARN Resource Manager. The client communicates with the - ResourceManager using the 'ApplicationClientProtocol' to first acquire a new - 'ApplicationId' if needed via ApplicationClientProtocol#getNewApplication and then - submit the 'Application' to be run via ApplicationClientProtocol#submitApplication. As - part of the ApplicationClientProtocol#submitApplication call, the client needs to - provide sufficient information to the ResourceManager to 'launch' the - application's first container i.e. the ApplicationMaster. - You need to provide information such as the details about the local - files/jars that need to be available for your application to run, the actual - command that needs to be executed (with the necessary command line arguments), - any Unix environment settings (optional), etc. Effectively, you need to - describe the Unix process(es) that needs to be launched for your - ApplicationMaster. + The general concept is that an submits an + to the YARN (RM). This can be done through + setting up a <<>> object. After <<>> is started, the + client can then set up application context, prepare the very first container of + the application that contains the (AM), and then submit + the application. You need to provide information such as the details about the + local files/jars that need to be available for your application to run, the + actual command that needs to be executed (with the necessary command line + arguments), any OS environment settings (optional), etc. Effectively, you + need to describe the Unix process(es) that needs to be launched for your + ApplicationMaster. - The YARN ResourceManager will then launch the ApplicationMaster (as specified) - on an allocated container. The ApplicationMaster is then expected to - communicate with the ResourceManager using the 'ApplicationMasterProtocol'. Firstly, the - ApplicationMaster needs to register itself with the ResourceManager. To - complete the task assigned to it, the ApplicationMaster can then request for - and receive containers via ApplicationMasterProtocol#allocate. After a container is - allocated to it, the ApplicationMaster communicates with the NodeManager using - ContainerManager#startContainer to launch the container for its task. As part - of launching this container, the ApplicationMaster has to specify the - ContainerLaunchContext which, similar to the ApplicationSubmissionContext, - has the launch information such as command line specification, environment, - etc. Once the task is completed, the ApplicationMaster has to signal the - ResourceManager of its completion via the ApplicationMasterProtocol#finishApplicationMaster. + The YARN ResourceManager will then launch the ApplicationMaster (as + specified) on an allocated container. The ApplicationMaster communicates with + YARN cluster, and handles application execution. It performs operations in an + asynchronous fashion. During application launch time, the main tasks of the + ApplicationMaster are: a) communicating with the ResourceManager to negotiate + and allocate resources for future containers, and b) after container + allocation, communicating YARN s (NMs) to launch application + containers on them. Task a) can be performed asynchronously through an + <<>> object, with event handling methods specified in a + <<>> type of event handler. The event handler + needs to be set to the client explicitly. Task b) can be performed by launching + a runnable object that then launches containers when there are containers + allocated. As part of launching this container, the AM has to + specify the <<>> that has the launch information such as + command line specification, environment, etc. - Meanwhile, the client can monitor the application's status by querying the - ResourceManager or by directly querying the ApplicationMaster if it supports - such a service. If needed, it can also kill the application via - ApplicationClientProtocol#forceKillApplication. + During the execution of an application, the ApplicationMaster communicates + NodeManagers through <<>> object. All container events are + handled by <<>>, associated with + <<>>. A typical callback handler handles client start, stop, + status update and error. ApplicationMaster also reports execution progress to + ResourceManager by handling the <<>> method of + <<>>. + + Other than asynchronous clients, there are synchronous versions for certain + workflows (<<>> and <<>>). The asynchronous clients are + recommended because of (subjectively) simpler usages, and this article + will mainly cover the asynchronous clients. Please refer to <<>> + and <<>> for more information on synchronous clients. -* Interfaces +* Interfaces The interfaces you'd most like be concerned with are: - * ApplicationClientProtocol - Client\<--\>ResourceManager\ - The protocol for a client that wishes to communicate with the - ResourceManager to launch a new application (i.e. the ApplicationMaster), - check on the status of the application or kill the application. For example, - a job-client (a job launching program from the gateway) would use this - protocol. + * <>\<--\><>\ + By using <<>> objects. + + * <>\<--\><>\ + By using <<>> objects, handling events asynchronously by + <<>> + + * <>\<--\><>\ + Launch containers. Communicate with NodeManagers + by using <<>> objects, handling container events by + <<>> + + [] + + <> - * ApplicationMasterProtocol - ApplicationMaster\<--\>ResourceManager\ - The protocol used by the ApplicationMaster to register/unregister itself - to/from the ResourceManager as well as to request for resources from the - Scheduler to complete its tasks. + * The three main protocols for YARN application (ApplicationClientProtocol, + ApplicationMasterProtocol and ContainerManagementProtocol) are still + preserved. The 3 clients wrap these 3 protocols to provide simpler + programming model for YARN applications. - * ContainerManager - ApplicationMaster\<--\>NodeManager\ - The protocol used by the ApplicationMaster to talk to the NodeManager to - start/stop containers and get status updates on the containers if needed. + * Under very rare circumstances, programmer may want to directly use the 3 + protocols to implement an application. However, note that . + + [] * Writing a Simple Yarn Application ** Writing a simple Client - * The first step that a client needs to do is to connect to the - ResourceManager or to be more specific, the ApplicationsManager (AsM) - interface of the ResourceManager. + * The first step that a client needs to do is to initialize and start a + YarnClient. +---+ - ApplicationClientProtocol applicationsManager; - YarnConfiguration yarnConf = new YarnConfiguration(conf); - InetSocketAddress rmAddress = - NetUtils.createSocketAddr(yarnConf.get( - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS)); - LOG.info("Connecting to ResourceManager at " + rmAddress); - configuration appsManagerServerConf = new Configuration(conf); - appsManagerServerConf.setClass( - YarnConfiguration.YARN_SECURITY_INFO, - ClientRMSecurityInfo.class, SecurityInfo.class); - applicationsManager = ((ApplicationClientProtocol) rpc.getProxy( - ApplicationClientProtocol.class, rmAddress, appsManagerServerConf)); + YarnClient yarnClient = YarnClient.createYarnClient(); + yarnClient.init(conf); + yarnClient.start(); +---+ - * Once a handle is obtained to the ASM, the client needs to request the - ResourceManager for a new ApplicationId. + * Once a client is set up, the client needs to create an application, and get + its application id. +---+ - GetNewApplicationRequest request = - Records.newRecord(GetNewApplicationRequest.class); - GetNewApplicationResponse response = - applicationsManager.getNewApplication(request); - LOG.info("Got new ApplicationId=" + response.getApplicationId()); + YarnClientApplication app = yarnClient.createApplication(); + GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); +---+ - * The response from the ASM for a new application also contains information - about the cluster such as the minimum/maximum resource capabilities of the - cluster. This is required so that to ensure that you can correctly set the - specifications of the container in which the ApplicationMaster would be - launched. Please refer to GetNewApplicationResponse for more details. + * The response from the <<>> for a new application also + contains information about the cluster such as the minimum/maximum resource + capabilities of the cluster. This is required so that to ensure that you can + correctly set the specifications of the container in which the + ApplicationMaster would be launched. Please refer to + <<>> for more details. - * The main crux of a client is to setup the ApplicationSubmissionContext - which defines all the information needed by the ResourceManager to launch - the ApplicationMaster. A client needs to set the following into the context: - - * Application Info: id, name + * The main crux of a client is to setup the <<>> + which defines all the information needed by the RM to launch the AM. A client + needs to set the following into the context: - * Queue, Priority info: Queue to which the application will be submitted, - the priority to be assigned for the application. + * Application info: id, name - * User: The user submitting the application + * Queue, priority info: Queue to which the application will be submitted, + the priority to be assigned for the application. - * ContainerLaunchContext: The information defining the container in which - the ApplicationMaster will be launched and run. The - ContainerLaunchContext, as mentioned previously, defines all the required - information needed to run the ApplicationMaster such as the local - resources (binaries, jars, files etc.), security tokens, environment - settings (CLASSPATH etc.) and the command to be executed. - - [] + * User: The user submitting the application + + * <<>>: The information defining the container in + which the AM will be launched and run. The <<>>, as + mentioned previously, defines all the required information needed to run + the application such as the local <>esources (binaries, jars, files + etc.), <>nvironment settings (CLASSPATH etc.), the <>ommand to be + executed and security <>okens (). + + [] +---+ - // Create a new ApplicationSubmissionContext - ApplicationSubmissionContext appContext = - Records.newRecord(ApplicationSubmissionContext.class); - // set the ApplicationId - appContext.setApplicationId(appId); - // set the application name - appContext.setApplicationName(appName); - - // Create a new container launch context for the AM's container - ContainerLaunchContext amContainer = - Records.newRecord(ContainerLaunchContext.class); + // set the application submission context + ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); - // Define the local resources required - Map localResources = - new HashMap(); - // Lets assume the jar we need for our ApplicationMaster is available in - // HDFS at a certain known path to us and we want to make it available to - // the ApplicationMaster in the launched container - Path jarPath; // <- known path to jar file - FileStatus jarStatus = fs.getFileStatus(jarPath); - LocalResource amJarRsrc = Records.newRecord(LocalResource.class); - // Set the type of resource - file or archive - // archives are untarred at the destination by the framework - amJarRsrc.setType(LocalResourceType.FILE); - // Set visibility of the resource - // Setting to most private option i.e. this file will only - // be visible to this instance of the running application - amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION); - // Set the location of resource to be copied over into the - // working directory - amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); - // Set timestamp and length of file so that the framework - // can do basic sanity checks for the local resource - // after it has been copied over to ensure it is the same - // resource the client intended to use with the application - amJarRsrc.setTimestamp(jarStatus.getModificationTime()); - amJarRsrc.setSize(jarStatus.getLen()); - // The framework will create a symlink called AppMaster.jar in the - // working directory that will be linked back to the actual file. - // The ApplicationMaster, if needs to reference the jar file, would - // need to use the symlink filename. - localResources.put("AppMaster.jar", amJarRsrc); - // Set the local resources into the launch context - amContainer.setLocalResources(localResources); + appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); + appContext.setApplicationName(appName); - // Set up the environment needed for the launch context - Map env = new HashMap(); - // For example, we could setup the classpath needed. - // Assuming our classes or jars are available as local resources in the - // working directory from which the command will be run, we need to append - // "." to the path. - // By default, all the hadoop specific classpaths will already be available - // in $CLASSPATH, so we should be careful not to overwrite it. - String classPathEnv = "$CLASSPATH:./*:"; - env.put("CLASSPATH", classPathEnv); - amContainer.setEnvironment(env); - - // Construct the command to be executed on the launched container - String command = - "${JAVA_HOME}" + /bin/java" + - " MyAppMaster" + - " arg1 arg2 arg3" + - " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + - " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; + // set local resources for the application master + // local files or archives as needed + // In this scenario, the jar file for the application master is part of the local resources + Map localResources = new HashMap(); - List commands = new ArrayList(); - commands.add(command); - // add additional commands if needed + LOG.info("Copy App Master jar from local filesystem and add to local environment"); + // Copy the application master jar to the filesystem + // Create a local resource to point to the destination jar path + FileSystem fs = FileSystem.get(conf); + addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), + localResources, null); - // Set the command array into the container spec - amContainer.setCommands(commands); - - // Define the resource requirements for the container - // For now, YARN only supports memory so we set the memory - // requirements. - // If the process takes more than its allocated memory, it will - // be killed by the framework. - // Memory being requested for should be less than max capability - // of the cluster and all asks should be a multiple of the min capability. - Resource capability = Records.newRecord(Resource.class); - capability.setMemory(amMemory); - amContainer.setResource(capability); - - // Set the container launch content into the ApplicationSubmissionContext - appContext.setAMContainerSpec(amContainer); -+---+ + // Set the log4j properties if needed + if (!log4jPropFile.isEmpty()) { + addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), + localResources, null); + } - * After the setup process is complete, the client is finally ready to submit - the application to the ASM. - -+---+ - // Create the request to send to the ApplicationsManager - SubmitApplicationRequest appRequest = - Records.newRecord(SubmitApplicationRequest.class); - appRequest.setApplicationSubmissionContext(appContext); + // The shell script has to be made available on the final container(s) + // where it will be executed. + // To do this, we need to first copy into the filesystem that is visible + // to the yarn framework. + // We do not need to set this as a local resource for the application + // master as the application master does not need it. + String hdfsShellScriptLocation = ""; + long hdfsShellScriptLen = 0; + long hdfsShellScriptTimestamp = 0; + if (!shellScriptPath.isEmpty()) { + Path shellSrc = new Path(shellScriptPath); + String shellPathSuffix = + appName + "/" + appId.toString() + "/" + SCRIPT_PATH; + Path shellDst = + new Path(fs.getHomeDirectory(), shellPathSuffix); + fs.copyFromLocalFile(false, true, shellSrc, shellDst); + hdfsShellScriptLocation = shellDst.toUri().toString(); + FileStatus shellFileStatus = fs.getFileStatus(shellDst); + hdfsShellScriptLen = shellFileStatus.getLen(); + hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); + } - // Submit the application to the ApplicationsManager - // Ignore the response as either a valid response object is returned on - // success or an exception thrown to denote the failure - applicationsManager.submitApplication(appRequest); -+---+ - - * At this point, the ResourceManager will have accepted the application and - in the background, will go through the process of allocating a container - with the required specifications and then eventually setting up and - launching the ApplicationMaster on the allocated container. - - * There are multiple ways a client can track progress of the actual task. - - * It can communicate with the ResourceManager and request for a report of - the application via ApplicationClientProtocol#getApplicationReport. + if (!shellCommand.isEmpty()) { + addToLocalResources(fs, null, shellCommandPath, appId.toString(), + localResources, shellCommand); + } -+-----+ - GetApplicationReportRequest reportRequest = - Records.newRecord(GetApplicationReportRequest.class); - reportRequest.setApplicationId(appId); - GetApplicationReportResponse reportResponse = - applicationsManager.getApplicationReport(reportRequest); - ApplicationReport report = reportResponse.getApplicationReport(); -+-----+ - - The ApplicationReport received from the ResourceManager consists of the following: - - * General application information: ApplicationId, queue to which the - application was submitted, user who submitted the application and the - start time for the application. - - * ApplicationMaster details: the host on which the ApplicationMaster is - running, the rpc port (if any) on which it is listening for requests - from clients and a token that the client needs to communicate with - the ApplicationMaster. - - * Application tracking information: If the application supports some - form of progress tracking, it can set a tracking url which is - available via ApplicationReport#getTrackingUrl that a client can look - at to monitor progress. - - * ApplicationStatus: The state of the application as seen by the - ResourceManager is available via - ApplicationReport#getYarnApplicationState. If the - YarnApplicationState is set to FINISHED, the client should refer to - ApplicationReport#getFinalApplicationStatus to check for the actual - success/failure of the application task itself. In case of failures, - ApplicationReport#getDiagnostics may be useful to shed some more - light on the the failure. - - * If the ApplicationMaster supports it, a client can directly query the - ApplicationMaster itself for progress updates via the host:rpcport - information obtained from the ApplicationReport. It can also use the - tracking url obtained from the report if available. + if (shellArgs.length > 0) { + addToLocalResources(fs, null, shellArgsPath, appId.toString(), + localResources, StringUtils.join(shellArgs, " ")); + } - * In certain situations, if the application is taking too long or due to - other factors, the client may wish to kill the application. The - ApplicationClientProtocol supports the forceKillApplication call that allows a - client to send a kill signal to the ApplicationMaster via the - ResourceManager. An ApplicationMaster if so designed may also support an - abort call via its rpc layer that a client may be able to leverage. + // Set the env variables to be setup in the env where the application master will be run + LOG.info("Set the environment for the application master"); + Map env = new HashMap(); -+---+ - KillApplicationRequest killRequest = - Records.newRecord(KillApplicationRequest.class); - killRequest.setApplicationId(appId); - applicationsManager.forceKillApplication(killRequest); -+---+ + // put location of shell script into env + // using the env info, the application master will create the correct local resource for the + // eventual containers that will be launched to execute the shell scripts + env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); + env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); + env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); -** Writing an ApplicationMaster + // Add AppMaster.jar location to classpath + // At some point we should not be required to add + // the hadoop specific classpaths to the env. + // It should be provided out of the box. + // For now setting all required classpaths including + // the classpath to "." for the application jar + StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) + .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); + for (String c : conf.getStrings( + YarnConfiguration.YARN_APPLICATION_CLASSPATH, + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { + classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); + classPathEnv.append(c.trim()); + } + classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append( + "./log4j.properties"); - * The ApplicationMaster is the actual owner of the job. It will be launched - by the ResourceManager and via the client will be provided all the necessary - information and resources about the job that it has been tasked with to - oversee and complete. + // Set the necessary command to execute the application master + Vector vargs = new Vector(30); - * As the ApplicationMaster is launched within a container that may (likely - will) be sharing a physical host with other containers, given the - multi-tenancy nature, amongst other issues, it cannot make any assumptions - of things like pre-configured ports that it can listen on. - - * When the ApplicationMaster starts up, several parameters are made available - to it via the environment. These include the ContainerId for the - ApplicationMaster container, the application submission time and details - about the NodeManager host running the Application Master. - Ref ApplicationConstants for parameter names. + // Set java executable command + LOG.info("Setting up app master command"); + vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); + // Set Xmx based on am memory size + vargs.add("-Xmx" + amMemory + "m"); + // Set class name + vargs.add(appMasterMainClass); + // Set params for Application Master + vargs.add("--container_memory " + String.valueOf(containerMemory)); + vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); + vargs.add("--num_containers " + String.valueOf(numContainers)); + vargs.add("--priority " + String.valueOf(shellCmdPriority)); - * All interactions with the ResourceManager require an ApplicationAttemptId - (there can be multiple attempts per application in case of failures). The - ApplicationAttemptId can be obtained from the ApplicationMaster - containerId. There are helper apis to convert the value obtained from the - environment into objects. - -+---+ - Map envs = System.getenv(); - String containerIdString = - envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV); - if (containerIdString == null) { - // container id should always be set in the env by the framework - throw new IllegalArgumentException( - "ContainerId not set in the environment"); + for (Map.Entry entry : shellEnv.entrySet()) { + vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); + } + if (debugFlag) { + vargs.add("--debug"); + } + + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); + + // Get final commmand + StringBuilder command = new StringBuilder(); + for (CharSequence str : vargs) { + command.append(str).append(" "); + } + + LOG.info("Completed setting up app master command " + command.toString()); + List commands = new ArrayList(); + commands.add(command.toString()); + + // Set up the container launch context for the application master + ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( + localResources, env, commands, null, null, null); + + // Set up resource type requirements + // For now, both memory and vcores are supported, so we set memory and + // vcores requirements + Resource capability = Resource.newInstance(amMemory, amVCores); + appContext.setResource(capability); + + // Service data is a binary blob that can be passed to the application + // Not needed in this scenario + // amContainer.setServiceData(serviceData); + + // Setup security tokens + if (UserGroupInformation.isSecurityEnabled()) { + // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce + Credentials credentials = new Credentials(); + String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); + if (tokenRenewer == null || tokenRenewer.length() == 0) { + throw new IOException( + "Can't get Master Kerberos principal for the RM to use as renewer"); } - ContainerId containerId = ConverterUtils.toContainerId(containerIdString); - ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId(); -+---+ - - * After an ApplicationMaster has initialized itself completely, it needs to - register with the ResourceManager via - ApplicationMasterProtocol#registerApplicationMaster. The ApplicationMaster always - communicate via the Scheduler interface of the ResourceManager. - -+---+ - // Connect to the Scheduler of the ResourceManager. - YarnConfiguration yarnConf = new YarnConfiguration(conf); - InetSocketAddress rmAddress = - NetUtils.createSocketAddr(yarnConf.get( - YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); - LOG.info("Connecting to ResourceManager at " + rmAddress); - ApplicationMasterProtocol resourceManager = - (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmAddress, conf); - // Register the AM with the RM - // Set the required info into the registration request: - // ApplicationAttemptId, - // host on which the app master is running - // rpc port on which the app master accepts requests from the client - // tracking url for the client to track app master progress - RegisterApplicationMasterRequest appMasterRequest = - Records.newRecord(RegisterApplicationMasterRequest.class); - appMasterRequest.setApplicationAttemptId(appAttemptID); - appMasterRequest.setHost(appMasterHostname); - appMasterRequest.setRpcPort(appMasterRpcPort); - appMasterRequest.setTrackingUrl(appMasterTrackingUrl); + // For now, only getting tokens for the default file-system. + final Token tokens[] = + fs.addDelegationTokens(tokenRenewer, credentials); + if (tokens != null) { + for (Token token : tokens) { + LOG.info("Got dt for " + fs.getUri() + "; " + token); + } + } + DataOutputBuffer dob = new DataOutputBuffer(); + credentials.writeTokenStorageToStream(dob); + ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + amContainer.setTokens(fsTokens); + } - // The registration response is useful as it provides information about the - // cluster. - // Similar to the GetNewApplicationResponse in the client, it provides - // information about the min/mx resource capabilities of the cluster that - // would be needed by the ApplicationMaster when requesting for containers. - RegisterApplicationMasterResponse response = - resourceManager.registerApplicationMaster(appMasterRequest); + appContext.setAMContainerSpec(amContainer); +---+ - - * The ApplicationMaster has to emit heartbeats to the ResourceManager to keep - it informed that the ApplicationMaster is alive and still running. The - timeout expiry interval at the ResourceManager is defined by a config - setting accessible via YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS with the - default being defined by YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS. - The ApplicationMasterProtocol#allocate calls to the ResourceManager count as heartbeats - as it also supports sending progress update information. Therefore, an - allocate call with no containers requested and progress information updated - if any is a valid way for making heartbeat calls to the ResourceManager. - - * Based on the task requirements, the ApplicationMaster can ask for a set of - containers to run its tasks on. The ApplicationMaster has to use the - ResourceRequest class to define the following container specifications: - - * Hostname: If containers are required to be hosted on a particular rack or - a specific host. '*' is a special value that implies any host will do. - - * Resource capability: Currently, YARN only supports memory based resource - requirements so the request should define how much memory is needed. The - value is defined in MB and has to less than the max capability of the + + * After the setup process is complete, the client is ready to submit + the application with specified priority and queue. + ++---+ + // Set the priority for the application master + Priority pri = Priority.newInstance(amPriority); + appContext.setPriority(pri); + + // Set the queue to which this application is to be submitted in the RM + appContext.setQueue(amQueue); + + // Submit the application to the applications manager + // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); + + yarnClient.submitApplication(appContext); ++---+ + + * At this point, the RM will have accepted the application and in the + background, will go through the process of allocating a container with the + required specifications and then eventually setting up and launching the AM + on the allocated container. + + * There are multiple ways a client can track progress of the actual task. + + * It can communicate with the RM and request for a report of the application + via the <<>> method of <<>>. + ++-----+ + // Get application report for the appId we are interested in + ApplicationReport report = yarnClient.getApplicationReport(appId); ++-----+ + + The <<>> received from the RM consists of the following: + + * General application information: Application id, queue to which the + application was submitted, user who submitted the application and the + start time for the application. + + * ApplicationMaster details: the host on which the AM is running, the + rpc port (if any) on which it is listening for requests from clients + and a token that the client needs to communicate with the AM. + + * Application tracking information: If the application supports some form + of progress tracking, it can set a tracking url which is available via + <<>>'s <<>> method that a client + can look at to monitor progress. + + * Application status: The state of the application as seen by the + ResourceManager is available via + <<>>. If the + <<>> is set to <<>>, the client should + refer to <<>> to check for + the actual success/failure of the application task itself. In case of + failures, <<>> may be useful to shed + some more light on the the failure. + + * If the ApplicationMaster supports it, a client can directly query the AM + itself for progress updates via the host:rpcport information obtained from + the application report. It can also use the tracking url obtained from the + report if available. + + * In certain situations, if the application is taking too long or due to other + factors, the client may wish to kill the application. <<>> + supports the <<>> call that allows a client to send a kill + signal to the AM via the ResourceManager. An ApplicationMaster if so + designed may also support an abort call via its rpc layer that a client may + be able to leverage. + ++---+ + yarnClient.killApplication(appId); ++---+ + +** Writing an ApplicationMaster (AM) + + * The AM is the actual owner of the job. It will be launched + by the RM and via the client will be provided all the + necessary information and resources about the job that it has been tasked + with to oversee and complete. + + * As the AM is launched within a container that may (likely + will) be sharing a physical host with other containers, given the + multi-tenancy nature, amongst other issues, it cannot make any assumptions + of things like pre-configured ports that it can listen on. + + * When the AM starts up, several parameters are made available + to it via the environment. These include the <<>> for the + AM container, the application submission time and details + about the NM (NodeManager) host running the ApplicationMaster. + Ref <<>> for parameter names. + + * All interactions with the RM require an <<>> (there can + be multiple attempts per application in case of failures). The + <<>> can be obtained from the AM's container id. There + are helper APIs to convert the value obtained from the environment into + objects. + ++---+ + Map envs = System.getenv(); + String containerIdString = + envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV); + if (containerIdString == null) { + // container id should always be set in the env by the framework + throw new IllegalArgumentException( + "ContainerId not set in the environment"); + } + ContainerId containerId = ConverterUtils.toContainerId(containerIdString); + ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId(); ++---+ + + * After an AM has initialized itself completely, we can start the two clients: + one to ResourceManager, and one to NodeManagers. We set them up with our + customized event handler, and we will talk about those event handlers in + detail later in this article. + ++---+ + AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); + amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); + amRMClient.init(conf); + amRMClient.start(); + + containerListener = createNMCallbackHandler(); + nmClientAsync = new NMClientAsyncImpl(containerListener); + nmClientAsync.init(conf); + nmClientAsync.start(); ++---+ + + * The AM has to emit heartbeats to the RM to keep it informed that the AM is + alive and still running. The timeout expiry interval at the RM is defined by + a config setting accessible via + <<>> with the default being + defined by <<>>. The + ApplicationMaster needs to register itself with the ResourceManager to + start hearbeating. + ++---+ + // Register self with ResourceManager + // This will start heartbeating to the RM + appMasterHostname = NetUtils.getHostname(); + RegisterApplicationMasterResponse response = amRMClient + .registerApplicationMaster(appMasterHostname, appMasterRpcPort, + appMasterTrackingUrl); ++---+ + + * In the response of the registration, maximum resource capability if included. You may want to use this to check the application's request. + ++---+ + // Dump out information about cluster capability as seen by the + // resource manager + int maxMem = response.getMaximumResourceCapability().getMemory(); + LOG.info("Max mem capabililty of resources in this cluster " + maxMem); + + int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); + LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); + + // A resource ask cannot exceed the max. + if (containerMemory > maxMem) { + LOG.info("Container memory specified above max threshold of cluster." + + " Using max value." + ", specified=" + containerMemory + ", max=" + + maxMem); + containerMemory = maxMem; + } + + if (containerVirtualCores > maxVCores) { + LOG.info("Container virtual cores specified above max threshold of cluster." + + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + + maxVCores); + containerVirtualCores = maxVCores; + } + List previousAMRunningContainers = + response.getContainersFromPreviousAttempts(); + LOG.info("Received " + previousAMRunningContainers.size() + + " previous AM's running containers on AM registration."); ++---+ + + * Based on the task requirements, the AM can ask for a set of containers to run + its tasks on. We can now calculate how many containers we need, and request + those many containers. + ++---+ + List previousAMRunningContainers = + response.getContainersFromPreviousAttempts(); + List previousAMRunningContainers = + response.getContainersFromPreviousAttempts(); + LOG.info("Received " + previousAMRunningContainers.size() + + " previous AM's running containers on AM registration."); + + int numTotalContainersToRequest = + numTotalContainers - previousAMRunningContainers.size(); + // Setup ask for containers from RM + // Send request for containers to RM + // Until we get our fully allocated quota, we keep on polling RM for + // containers + // Keep looping until all the containers are launched and shell script + // executed on them ( regardless of success/failure). + for (int i = 0; i < numTotalContainersToRequest; ++i) { + ContainerRequest containerAsk = setupContainerAskForRM(); + amRMClient.addContainerRequest(containerAsk); + } ++---+ + + * In <<>>, the follow two things need some set up: + + * Resource capability: Currently, YARN supports memory based resource + requirements so the request should define how much memory is needed. The + value is defined in MB and has to less than the max capability of the cluster and an exact multiple of the min capability. Memory resources - correspond to physical memory limits imposed on the task containers. - - * Priority: When asking for sets of containers, an ApplicationMaster may - define different priorities to each set. For example, the Map-Reduce - ApplicationMaster may assign a higher priority to containers needed - for the Map tasks and a lower priority for the Reduce tasks' containers. - - [] - -+----+ - // Resource Request - ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class); + correspond to physical memory limits imposed on the task containers. It + will also support computation based resource (vCore), as shown in the code. - // setup requirements for hosts - // whether a particular rack/host is needed - // useful for applications that are sensitive - // to data locality - rsrcRequest.setHostName("*"); + * Priority: When asking for sets of containers, an AM may define different + priorities to each set. For example, the Map-Reduce AM may assign a higher + priority to containers needed for the Map tasks and a lower priority for + the Reduce tasks' containers. + [] + ++---+ + private ContainerRequest setupContainerAskForRM() { + // setup requirements for hosts + // using * as any host will do for the distributed shell app // set the priority for the request - Priority pri = Records.newRecord(Priority.class); - pri.setPriority(requestPriority); - rsrcRequest.setPriority(pri); + Priority pri = Priority.newInstance(requestPriority); // Set up resource type requirements - // For now, only memory is supported so we set memory requirements - Resource capability = Records.newRecord(Resource.class); - capability.setMemory(containerMemory); - rsrcRequest.setCapability(capability); - - // set no. of containers needed - // matching the specifications - rsrcRequest.setNumContainers(numContainers); -+---+ - - * After defining the container requirements, the ApplicationMaster has to - construct an AllocateRequest to send to the ResourceManager. - The AllocateRequest consists of: - - * Requested containers: The container specifications and the no. of - containers being requested for by the ApplicationMaster from the - ResourceManager. - - * Released containers: There may be situations when the ApplicationMaster - may have requested for more containers that it needs or due to failure - issues, decide to use other containers allocated to it. In all such - situations, it is beneficial to the cluster if the ApplicationMaster - releases these containers back to the ResourceManager so that they can be - re-allocated to other applications. - - * ResponseId: The response id that will be sent back in the response from - the allocate call. - - * Progress update information: The ApplicationMaster can send its progress - update (range between to 0 to 1) to the ResourceManager. - - [] - -+---+ - List requestedContainers; - List releasedContainers - AllocateRequest req = Records.newRecord(AllocateRequest.class); - - // The response id set in the request will be sent back in - // the response so that the ApplicationMaster can - // match it to its original ask and act appropriately. - req.setResponseId(rmRequestID); - - // Set ApplicationAttemptId - req.setApplicationAttemptId(appAttemptID); - - // Add the list of containers being asked for - req.addAllAsks(requestedContainers); - - // If the ApplicationMaster has no need for certain - // containers due to over-allocation or for any other - // reason, it can release them back to the ResourceManager - req.addAllReleases(releasedContainers); - - // Assuming the ApplicationMaster can track its progress - req.setProgress(currentProgress); - - AllocateResponse allocateResponse = resourceManager.allocate(req); -+---+ - - * The AllocateResponse sent back from the ResourceManager provides the - following information: - - * Reboot flag: For scenarios when the ApplicationMaster may get out of sync - with the ResourceManager. - - * Allocated containers: The containers that have been allocated to the - ApplicationMaster. - - * Headroom: Headroom for resources in the cluster. Based on this information - and knowing its needs, an ApplicationMaster can make intelligent decisions - such as re-prioritizing sub-tasks to take advantage of currently allocated - containers, bailing out faster if resources are not becoming available - etc. - - * Completed containers: Once an ApplicationMaster triggers a launch an - allocated container, it will receive an update from the ResourceManager - when the container completes. The ApplicationMaster can look into the - status of the completed container and take appropriate actions such as - re-trying a particular sub-task in case of a failure. - - * Number of cluster nodes: The number of hosts available on the cluster. - - [] - - One thing to note is that containers will not be immediately allocated to - the ApplicationMaster. This does not imply that the ApplicationMaster should - keep on asking the pending count of required containers. Once an allocate - request has been sent, the ApplicationMaster will eventually be allocated - the containers based on cluster capacity, priorities and the scheduling - policy in place. The ApplicationMaster should only request for containers - again if and only if its original estimate changed and it needs additional - containers. + // For now, memory and CPU are supported so we set memory and cpu requirements + Resource capability = Resource.newInstance(containerMemory, + containerVirtualCores); + ContainerRequest request = new ContainerRequest(capability, null, null, + pri); + LOG.info("Requested container ask: " + request.toString()); + return request; + } +---+ - // Retrieve list of allocated containers from the response - // and on each allocated container, lets assume we are launching - // the same job. - List allocatedContainers = allocateResponse.getAllocatedContainers(); + * After container allocation requests have been sent by the application + manager, contailers will be launched asynchronously, by the event handler of + the <<>> client. The handler should implement + <<>> interface. + + * When there are containers allocated, the handler sets up a thread that runs + the code to launch containers. Here we use the name + <<>> to demonstrate. We will talk about the + <<>> class in the following part of this article. + ++---+ + @Override + public void onContainersAllocated(List allocatedContainers) { + LOG.info("Got response from RM for container ask, allocatedCnt=" + + allocatedContainers.size()); + numAllocatedContainers.addAndGet(allocatedContainers.size()); for (Container allocatedContainer : allocatedContainers) { - LOG.info("Launching shell command on a new container." - + ", containerId=" + allocatedContainer.getId() - + ", containerNode=" + allocatedContainer.getNodeId().getHost() - + ":" + allocatedContainer.getNodeId().getPort() - + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() - + ", containerState" + allocatedContainer.getState() - + ", containerResourceMemory" - + allocatedContainer.getResource().getMemory()); - - - // Launch and start the container on a separate thread to keep the main - // thread unblocked as all containers may not be allocated at one go. - LaunchContainerRunnable runnableLaunchContainer = - new LaunchContainerRunnable(allocatedContainer); - Thread launchThread = new Thread(runnableLaunchContainer); + LaunchContainerRunnable runnableLaunchContainer = + new LaunchContainerRunnable(allocatedContainer, containerListener); + Thread launchThread = new Thread(runnableLaunchContainer); + + // launch and start the container on a separate thread to keep + // the main thread unblocked + // as all containers may not be allocated at one go. launchThreads.add(launchThread); launchThread.start(); } - - // Check what the current available resources in the cluster are - Resource availableResources = allocateResponse.getAvailableResources(); - // Based on this information, an ApplicationMaster can make appropriate - // decisions - - // Check the completed containers - // Let's assume we are keeping a count of total completed containers, - // containers that failed and ones that completed successfully. - List completedContainers = - allocateResponse.getCompletedContainersStatuses(); - for (ContainerStatus containerStatus : completedContainers) { - LOG.info("Got container status for containerID= " - + containerStatus.getContainerId() - + ", state=" + containerStatus.getState() - + ", exitStatus=" + containerStatus.getExitStatus() - + ", diagnostics=" + containerStatus.getDiagnostics()); - - int exitStatus = containerStatus.getExitStatus(); - if (0 != exitStatus) { - // container failed - // -100 is a special case where the container - // was aborted/pre-empted for some reason - if (-100 != exitStatus) { - // application job on container returned a non-zero exit code - // counts as completed - numCompletedContainers.incrementAndGet(); - numFailedContainers.incrementAndGet(); - } - else { - // something else bad happened - // app job did not complete for some reason - // we should re-try as the container was lost for some reason - // decrementing the requested count so that we ask for an - // additional one in the next allocate call. - numRequestedContainers.decrementAndGet(); - // we do not need to release the container as that has already - // been done by the ResourceManager/NodeManager. - } - } - else { - // nothing to do - // container completed successfully - numCompletedContainers.incrementAndGet(); - numSuccessfulContainers.incrementAndGet(); - } - } - } -+---+ - - - * After a container has been allocated to the ApplicationMaster, it needs to - follow a similar process that the Client followed in setting up the - ContainerLaunchContext for the eventual task that is going to be running on - the allocated Container. Once the ContainerLaunchContext is defined, the - ApplicationMaster can then communicate with the ContainerManager to start - its allocated container. - -+---+ - - //Assuming an allocated Container obtained from AllocateResponse - Container container; - // Connect to ContainerManager on the allocated container - String cmIpPortStr = container.getNodeId().getHost() + ":" - + container.getNodeId().getPort(); - InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr); - ContainerManager cm = - (ContainerManager)rpc.getProxy(ContainerManager.class, cmAddress, conf); - - // Now we setup a ContainerLaunchContext - ContainerLaunchContext ctx = - Records.newRecord(ContainerLaunchContext.class); - - ctx.setContainerId(container.getId()); - ctx.setResource(container.getResource()); - - try { - ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); - } catch (IOException e) { - LOG.info( - "Getting current user failed when trying to launch the container", - + e.getMessage()); - } - - // Set the environment - Map unixEnv; - // Setup the required env. - // Please note that the launched container does not inherit - // the environment of the ApplicationMaster so all the - // necessary environment settings will need to be re-setup - // for this allocated container. - ctx.setEnvironment(unixEnv); - - // Set the local resources - Map localResources = - new HashMap(); - // Again, the local resources from the ApplicationMaster is not copied over - // by default to the allocated container. Thus, it is the responsibility - // of the ApplicationMaster to setup all the necessary local resources - // needed by the job that will be executed on the allocated container. - - // Assume that we are executing a shell script on the allocated container - // and the shell script's location in the filesystem is known to us. - Path shellScriptPath; - LocalResource shellRsrc = Records.newRecord(LocalResource.class); - shellRsrc.setType(LocalResourceType.FILE); - shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION); - shellRsrc.setResource( - ConverterUtils.getYarnUrlFromURI(new URI(shellScriptPath))); - shellRsrc.setTimestamp(shellScriptPathTimestamp); - shellRsrc.setSize(shellScriptPathLen); - localResources.put("MyExecShell.sh", shellRsrc); - - ctx.setLocalResources(localResources); - - // Set the necessary command to execute on the allocated container - String command = "/bin/sh ./MyExecShell.sh" - + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" - + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; - - List commands = new ArrayList(); - commands.add(command); - ctx.setCommands(commands); - - // Send the start request to the ContainerManager - StartContainerRequest startReq = Records.newRecord(StartContainerRequest.class); - startReq.setContainerLaunchContext(ctx); - cm.startContainer(startReq); -+---+ - - * The ApplicationMaster, as mentioned previously, will get updates of - completed containers as part of the response from the ApplicationMasterProtocol#allocate - calls. It can also monitor its launched containers pro-actively by querying - the ContainerManager for the status. - + } +---+ - GetContainerStatusRequest statusReq = - Records.newRecord(GetContainerStatusRequest.class); - statusReq.setContainerId(container.getId()); - GetContainerStatusResponse statusResp = cm.getContainerStatus(statusReq); - LOG.info("Container Status" - + ", id=" + container.getId() - + ", status=" + statusResp.getStatus()); -+---+ + * On heart beat, the event handler reports the progress of the application. + ++---+ + @Override + public float getProgress() { + // set progress to deliver to RM on next heartbeat + float progress = (float) numCompletedContainers.get() + / numTotalContainers; + return progress; + } ++---+ + + [] + + * The container launch thread actually launches the containers on NMs. After a + container has been allocated to the AM, it needs to follow a similar process + that the client followed in setting up the <<>> for + the eventual task that is going to be running on the allocated Container. + Once the <<>> is defined, the AM can start it through + the <<>>. + ++---+ + // Set the necessary command to execute on the allocated container + Vector vargs = new Vector(5); + + // Set executable command + vargs.add(shellCommand); + // Set shell script path + if (!scriptPath.isEmpty()) { + vargs.add(Shell.WINDOWS ? ExecBatScripStringtPath + : ExecShellStringPath); + } + + // Set args for the shell command if any + vargs.add(shellArgs); + // Add log redirect params + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); + + // Get final commmand + StringBuilder command = new StringBuilder(); + for (CharSequence str : vargs) { + command.append(str).append(" "); + } + + List commands = new ArrayList(); + commands.add(command.toString()); + + // Set up ContainerLaunchContext, setting local resource, environment, + // command and token for constructor. + + // Note for tokens: Set up tokens for the container too. Today, for normal + // shell commands, the container in distribute-shell doesn't need any + // tokens. We are populating them mainly for NodeManagers to be able to + // download anyfiles in the distributed file-system. The tokens are + // otherwise also useful in cases, for e.g., when one is running a + // "hadoop dfs" command inside the distributed shell. + ContainerLaunchContext ctx = ContainerLaunchContext.newInstance( + localResources, shellEnv, commands, null, allTokens.duplicate(), null); + containerListener.addContainer(container.getId(), container); + nmClientAsync.startContainerAsync(container, ctx); ++---+ + + * The <<>> object, together with its event handler, handles container events. Including container start, stop, status update, and occurs an error. + + * After the ApplicationMaster determines the work is done, it needs to unregister itself through the AM-RM client, and then stops the client. + ++---+ + try { + amRMClient.unregisterApplicationMaster(appStatus, appMessage, null); + } catch (YarnException ex) { + LOG.error("Failed to unregister application", ex); + } catch (IOException e) { + LOG.error("Failed to unregister application", e); + } + + amRMClient.stop(); ++---+ ~~** Defining the context in which your code runs -~~*** Container Resource Requests +~~*** Container Resource Requests -~~*** Local Resources +~~*** Local Resources -~~*** Environment +~~*** Environment -~~**** Managing the CLASSPATH +~~**** Managing the CLASSPATH -~~** Security +~~** Security -* FAQ +* FAQ -** How can I distribute my application's jars to all of the nodes in the YARN +** How can I distribute my application's jars to all of the nodes in the YARN cluster that need it? - You can use the LocalResource to add resources to your application request. - This will cause YARN to distribute the resource to the ApplicationMaster node. - If the resource is a tgz, zip, or jar - you can have YARN unzip it. Then, all - you need to do is add the unzipped folder to your classpath. - For example, when creating your application request: + * You can use the LocalResource to add resources to your application request. + This will cause YARN to distribute the resource to the ApplicationMaster + node. If the resource is a tgz, zip, or jar - you can have YARN unzip it. + Then, all you need to do is add the unzipped folder to your classpath. For + example, when creating your application request: +---+ - File packageFile = new File(packagePath); - Url packageUrl = ConverterUtils.getYarnUrlFromPath( - FileContext.getFileContext.makeQualified(new Path(packagePath))); + File packageFile = new File(packagePath); + Url packageUrl = ConverterUtils.getYarnUrlFromPath( + FileContext.getFileContext.makeQualified(new Path(packagePath))); - packageResource.setResource(packageUrl); - packageResource.setSize(packageFile.length()); - packageResource.setTimestamp(packageFile.lastModified()); - packageResource.setType(LocalResourceType.ARCHIVE); - packageResource.setVisibility(LocalResourceVisibility.APPLICATION); + packageResource.setResource(packageUrl); + packageResource.setSize(packageFile.length()); + packageResource.setTimestamp(packageFile.lastModified()); + packageResource.setType(LocalResourceType.ARCHIVE); + packageResource.setVisibility(LocalResourceVisibility.APPLICATION); - resource.setMemory(memory) - containerCtx.setResource(resource) - containerCtx.setCommands(ImmutableList.of( - "java -cp './package/*' some.class.to.Run " - + "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout " - + "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")) - containerCtx.setLocalResources( - Collections.singletonMap("package", packageResource)) - appCtx.setApplicationId(appId) - appCtx.setUser(user.getShortUserName) - appCtx.setAMContainerSpec(containerCtx) - request.setApplicationSubmissionContext(appCtx) - applicationsManager.submitApplication(request) + resource.setMemory(memory); + containerCtx.setResource(resource); + containerCtx.setCommands(ImmutableList.of( + "java -cp './package/*' some.class.to.Run " + + "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout " + + "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); + containerCtx.setLocalResources( + Collections.singletonMap("package", packageResource)); + appCtx.setApplicationId(appId); + appCtx.setUser(user.getShortUserName); + appCtx.setAMContainerSpec(containerCtx); + yarnClient.submitApplication(appCtx); +---+ - As you can see, the setLocalResources command takes a map of names to - resources. The name becomes a sym link in your application's cwd, so you can - just refer to the artifacts inside by using ./package/*. - - Note: Java's classpath (cp) argument is VERY sensitive. + As you can see, the <<>> command takes a map of names to + resources. The name becomes a sym link in your application's cwd, so you can + just refer to the artifacts inside by using ./package/*. + + Note: Java's classpath (cp) argument is VERY sensitive. Make sure you get the syntax EXACTLY correct. - Once your package is distributed to your ApplicationMaster, you'll need to - follow the same process whenever your ApplicationMaster starts a new container - (assuming you want the resources to be sent to your container). The code for - this is the same. You just need to make sure that you give your - ApplicationMaster the package path (either HDFS, or local), so that it can - send the resource URL along with the container ctx. + Once your package is distributed to your AM, you'll need to follow the same + process whenever your AM starts a new container (assuming you want the + resources to be sent to your container). The code for this is the same. You + just need to make sure that you give your AM the package path (either HDFS, or + local), so that it can send the resource URL along with the container ctx. -** How do I get the ApplicationMaster's ApplicationAttemptId? +** How do I get the ApplicationMaster's <<>>? + * The <<>> will be passed to the AM via the environment + and the value from the environment can be converted into an + <<>> object via the ConverterUtils helper function. - The ApplicationAttemptId will be passed to the ApplicationMaster via the - environment and the value from the environment can be converted into an - ApplicationAttemptId object via the ConverterUtils helper function. +** Why my container is killed by the NodeManager? -** My container is being killed by the Node Manager - - This is likely due to high memory usage exceeding your requested container - memory size. There are a number of reasons that can cause this. First, look - at the process tree that the node manager dumps when it kills your container. - The two things you're interested in are physical memory and virtual memory. - If you have exceeded physical memory limits your app is using too much physical - memory. If you're running a Java app, you can use -hprof to look at what is - taking up space in the heap. If you have exceeded virtual memory, you may - need to increase the value of the the cluster-wide configuration variable - <<>>. + * This is likely due to high memory usage exceeding your requested container + memory size. There are a number of reasons that can cause this. First, look + at the process tree that the NodeManager dumps when it kills your container. + The two things you're interested in are physical memory and virtual memory. + If you have exceeded physical memory limits your app is using too much + physical memory. If you're running a Java app, you can use -hprof to look at + what is taking up space in the heap. If you have exceeded virtual memory, you + may need to increase the value of the the cluster-wide configuration variable + <<>>. ** How do I include native libraries? - - Setting -Djava.library.path on the command line while launching a container - can cause native libraries used by Hadoop to not be loaded correctly and can - result in errors. It is cleaner to use LD_LIBRARY_PATH instead. + * Setting <<<-Djava.library.path>>> on the command line while launching a + container can cause native libraries used by Hadoop to not be loaded + correctly and can result in errors. It is cleaner to use + <<>> instead. * Useful Links - * {{{https://issues.apache.org/jira/secure/attachment/12486023/MapReduce_NextGen_Architecture.pdf}Map Reduce Next Generation Architecture}} + * {{{http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html}YARN Architecture}} - * {{{http://developer.yahoo.com/blogs/hadoop/posts/2011/03/mapreduce-nextgen-scheduler/}Map Reduce Next Generation Scheduler}} + * {{{http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html}YARN Capacity Scheduler}} + + * {{{http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/FairScheduler.html}YARN Fair Scheduler}} + +* Sample code + + * Yarn distributed shell: in <<>> + project after you set up your development environment. From 662c5bb3af474a78688d0df523d11aff7ee756b6 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 12 Aug 2014 21:42:21 +0000 Subject: [PATCH 04/67] HDFS-6830. BlockInfo.addStorage fails when DN changes the storage for a block replica. (Arpit Agarwal) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617598 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/blockmanagement/BlockInfo.java | 26 ++++++------------- .../server/blockmanagement/BlockManager.java | 4 +-- .../blockmanagement/DatanodeStorageInfo.java | 22 +++++++++++++--- .../server/blockmanagement/TestBlockInfo.java | 20 +++++++++----- 5 files changed, 46 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 65253c299f7..e448ca6727d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -491,6 +491,9 @@ Release 2.6.0 - UNRELEASED HDFS-6582. Missing null check in RpcProgramNfs3#read(XDR, SecurityHandler) (Abhiraj Butala via brandonli) + HDFS-6830. BlockInfo.addStorage fails when DN changes the storage for a + block replica (Arpit Agarwal) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index 272cb1d48a1..0ce112118c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -194,24 +194,12 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { * Add a {@link DatanodeStorageInfo} location for a block */ boolean addStorage(DatanodeStorageInfo storage) { - boolean added = true; - int idx = findDatanode(storage.getDatanodeDescriptor()); - if(idx >= 0) { - if (getStorageInfo(idx) == storage) { // the storage is already there - return false; - } else { - // The block is on the DN but belongs to a different storage. - // Update our state. - removeStorage(getStorageInfo(idx)); - added = false; // Just updating storage. Return false. - } - } // find the last null node int lastNode = ensureCapacity(1); setStorageInfo(lastNode, storage); setNext(lastNode, null); setPrevious(lastNode, null); - return added; + return true; } /** @@ -240,16 +228,18 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { * Find specified DatanodeDescriptor. * @return index or -1 if not found. */ - int findDatanode(DatanodeDescriptor dn) { + boolean findDatanode(DatanodeDescriptor dn) { int len = getCapacity(); for(int idx = 0; idx < len; idx++) { DatanodeDescriptor cur = getDatanode(idx); - if(cur == dn) - return idx; - if(cur == null) + if(cur == dn) { + return true; + } + if(cur == null) { break; + } } - return -1; + return false; } /** * Find specified DatanodeStorageInfo. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 8046d8ad0cd..6fdec77be69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2065,7 +2065,7 @@ public class BlockManager { // Add replica if appropriate. If the replica was previously corrupt // but now okay, it might need to be updated. if (reportedState == ReplicaState.FINALIZED - && (storedBlock.findDatanode(dn) < 0 + && (!storedBlock.findDatanode(dn) || corruptReplicas.isReplicaCorrupt(storedBlock, dn))) { toAdd.add(storedBlock); } @@ -2246,7 +2246,7 @@ public class BlockManager { storageInfo, ucBlock.reportedBlock, ucBlock.reportedState); if (ucBlock.reportedState == ReplicaState.FINALIZED && - block.findDatanode(storageInfo.getDatanodeDescriptor()) < 0) { + !block.findDatanode(storageInfo.getDatanodeDescriptor())) { addStoredBlock(block, storageInfo, null, true); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 791fc3157d9..9e442c7e1f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -208,12 +208,28 @@ public class DatanodeStorageInfo { } public boolean addBlock(BlockInfo b) { - if(!b.addStorage(this)) - return false; + // First check whether the block belongs to a different storage + // on the same DN. + boolean replaced = false; + DatanodeStorageInfo otherStorage = + b.findStorageInfo(getDatanodeDescriptor()); + + if (otherStorage != null) { + if (otherStorage != this) { + // The block belongs to a different storage. Remove it first. + otherStorage.removeBlock(b); + replaced = true; + } else { + // The block is already associated with this storage. + return false; + } + } + // add to the head of the data-node list + b.addStorage(this); blockList = b.listInsert(blockList, this); numBlocks++; - return true; + return !replaced; } boolean removeBlock(BlockInfo b) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java index 7cfe423c6e4..61094dfc8ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -59,17 +60,24 @@ public class TestBlockInfo { @Test - public void testReplaceStorageIfDifferetnOneAlreadyExistedFromSameDataNode() throws Exception { - BlockInfo blockInfo = new BlockInfo(3); + public void testReplaceStorage() throws Exception { + // Create two dummy storages. final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1"); final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2")); + final int NUM_BLOCKS = 10; + BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS]; - blockInfo.addStorage(storage1); - boolean added = blockInfo.addStorage(storage2); + // Create a few dummy blocks and add them to the first storage. + for (int i = 0; i < NUM_BLOCKS; ++i) { + blockInfos[i] = new BlockInfo(3); + storage1.addBlock(blockInfos[i]); + } - Assert.assertFalse(added); - Assert.assertEquals(storage2, blockInfo.getStorageInfo(0)); + // Try to move one of the blocks to a different storage. + boolean added = storage2.addBlock(blockInfos[NUM_BLOCKS/2]); + Assert.assertThat(added, is(false)); + Assert.assertThat(blockInfos[NUM_BLOCKS/2].getStorageInfo(0), is(storage2)); } @Test From 486e718fc1f5befd231494e2ec06bb360484f191 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Tue, 12 Aug 2014 21:43:27 +0000 Subject: [PATCH 05/67] YARN-2399. FairScheduler: Merge AppSchedulable and FSSchedulerApp into FSAppAttempt. (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617600 13f79535-47bb-0310-9956-ffa450edef68 --- .../sls/scheduler/FairSchedulerMetrics.java | 8 +- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FSAppAttempt.java | 768 ++++++++++++++++++ .../scheduler/fair/FSLeafQueue.java | 66 +- .../scheduler/fair/FSQueue.java | 24 +- .../scheduler/fair/FSSchedulerNode.java | 6 +- .../scheduler/fair/FairScheduler.java | 68 +- .../scheduler/fair/FifoAppComparator.java | 8 +- .../fair/MaxRunningAppsEnforcer.java | 52 +- .../scheduler/fair/NewAppWeightBooster.java | 2 +- .../scheduler/fair/Schedulable.java | 70 +- .../scheduler/fair/WeightAdjuster.java | 2 +- .../webapp/dao/FairSchedulerInfo.java | 3 +- .../dao/FairSchedulerLeafQueueInfo.java | 9 +- .../scheduler/fair/FakeSchedulable.java | 18 +- .../scheduler/fair/TestFSAppAttempt.java | 188 +++++ .../scheduler/fair/TestFSLeafQueue.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 109 ++- .../fair/TestMaxRunningAppsEnforcer.java | 42 +- 19 files changed, 1199 insertions(+), 249 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java index f427dcd557a..d100e1de583 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java @@ -22,10 +22,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .AppSchedulable; + .FSAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import com.codahale.metrics.Gauge; import org.apache.hadoop.yarn.sls.SLSRunner; @@ -66,8 +65,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics { public void trackApp(ApplicationAttemptId appAttemptId, String oldAppId) { super.trackApp(appAttemptId, oldAppId); FairScheduler fair = (FairScheduler) scheduler; - final AppSchedulable app = fair.getSchedulerApp(appAttemptId) - .getAppSchedulable(); + final FSAppAttempt app = fair.getSchedulerApp(appAttemptId); metrics.register("variable.app." + oldAppId + ".demand.memory", new Gauge() { @Override diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index af0f9b56e32..d977127cd9d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -122,6 +122,9 @@ Release 2.6.0 - UNRELEASED YARN-2317. Updated the document about how to write YARN applications. (Li Lu via zjshen) + YARN-2399. FairScheduler: Merge AppSchedulable and FSSchedulerApp into + FSAppAttempt. (kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java new file mode 100644 index 00000000000..eb6f6413893 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -0,0 +1,768 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; + +/** + * Represents an application attempt from the viewpoint of the Fair Scheduler. + */ +@Private +@Unstable +public class FSAppAttempt extends SchedulerApplicationAttempt + implements Schedulable { + + private static final Log LOG = LogFactory.getLog(FSAppAttempt.class); + private static final DefaultResourceCalculator RESOURCE_CALCULATOR + = new DefaultResourceCalculator(); + + private long startTime; + private Priority priority; + private ResourceWeights resourceWeights; + private Resource demand = Resources.createResource(0); + private FairScheduler scheduler; + private Resource fairShare = Resources.createResource(0, 0); + private Resource preemptedResources = Resources.createResource(0); + private RMContainerComparator comparator = new RMContainerComparator(); + private final Map preemptionMap = new HashMap(); + + /** + * Delay scheduling: We often want to prioritize scheduling of node-local + * containers over rack-local or off-switch containers. To acheive this + * we first only allow node-local assigments for a given prioirty level, + * then relax the locality threshold once we've had a long enough period + * without succesfully scheduling. We measure both the number of "missed" + * scheduling opportunities since the last container was scheduled + * at the current allowed level and the time since the last container + * was scheduled. Currently we use only the former. + */ + private final Map allowedLocalityLevel = + new HashMap(); + + public FSAppAttempt(FairScheduler scheduler, + ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue, + ActiveUsersManager activeUsersManager, RMContext rmContext) { + super(applicationAttemptId, user, queue, activeUsersManager, rmContext); + + this.scheduler = scheduler; + this.startTime = scheduler.getClock().getTime(); + this.priority = Priority.newInstance(1); + this.resourceWeights = new ResourceWeights(); + } + + public ResourceWeights getResourceWeights() { + return resourceWeights; + } + + /** + * Get metrics reference from containing queue. + */ + public QueueMetrics getMetrics() { + return queue.getMetrics(); + } + + synchronized public void containerCompleted(RMContainer rmContainer, + ContainerStatus containerStatus, RMContainerEventType event) { + + Container container = rmContainer.getContainer(); + ContainerId containerId = container.getId(); + + // Remove from the list of newly allocated containers if found + newlyAllocatedContainers.remove(rmContainer); + + // Inform the container + rmContainer.handle( + new RMContainerFinishedEvent( + containerId, + containerStatus, + event) + ); + LOG.info("Completed container: " + rmContainer.getContainerId() + + " in state: " + rmContainer.getState() + " event:" + event); + + // Remove from the list of containers + liveContainers.remove(rmContainer.getContainerId()); + + RMAuditLogger.logSuccess(getUser(), + AuditConstants.RELEASE_CONTAINER, "SchedulerApp", + getApplicationId(), containerId); + + // Update usage metrics + Resource containerResource = rmContainer.getContainer().getResource(); + queue.getMetrics().releaseResources(getUser(), 1, containerResource); + Resources.subtractFrom(currentConsumption, containerResource); + + // remove from preemption map if it is completed + preemptionMap.remove(rmContainer); + } + + private synchronized void unreserveInternal( + Priority priority, FSSchedulerNode node) { + Map reservedContainers = + this.reservedContainers.get(priority); + RMContainer reservedContainer = reservedContainers.remove(node.getNodeID()); + if (reservedContainers.isEmpty()) { + this.reservedContainers.remove(priority); + } + + // Reset the re-reservation count + resetReReservations(priority); + + Resource resource = reservedContainer.getContainer().getResource(); + Resources.subtractFrom(currentReservation, resource); + + LOG.info("Application " + getApplicationId() + " unreserved " + " on node " + + node + ", currently has " + reservedContainers.size() + " at priority " + + priority + "; currentReservation " + currentReservation); + } + + public synchronized float getLocalityWaitFactor( + Priority priority, int clusterNodes) { + // Estimate: Required unique resources (i.e. hosts + racks) + int requiredResources = + Math.max(this.getResourceRequests(priority).size() - 1, 0); + + // waitFactor can't be more than '1' + // i.e. no point skipping more than clustersize opportunities + return Math.min(((float)requiredResources / clusterNodes), 1.0f); + } + + /** + * Return the level at which we are allowed to schedule containers, given the + * current size of the cluster and thresholds indicating how many nodes to + * fail at (as a fraction of cluster size) before relaxing scheduling + * constraints. + */ + public synchronized NodeType getAllowedLocalityLevel(Priority priority, + int numNodes, double nodeLocalityThreshold, double rackLocalityThreshold) { + // upper limit on threshold + if (nodeLocalityThreshold > 1.0) { nodeLocalityThreshold = 1.0; } + if (rackLocalityThreshold > 1.0) { rackLocalityThreshold = 1.0; } + + // If delay scheduling is not being used, can schedule anywhere + if (nodeLocalityThreshold < 0.0 || rackLocalityThreshold < 0.0) { + return NodeType.OFF_SWITCH; + } + + // Default level is NODE_LOCAL + if (!allowedLocalityLevel.containsKey(priority)) { + allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL); + return NodeType.NODE_LOCAL; + } + + NodeType allowed = allowedLocalityLevel.get(priority); + + // If level is already most liberal, we're done + if (allowed.equals(NodeType.OFF_SWITCH)) return NodeType.OFF_SWITCH; + + double threshold = allowed.equals(NodeType.NODE_LOCAL) ? nodeLocalityThreshold : + rackLocalityThreshold; + + // Relax locality constraints once we've surpassed threshold. + if (getSchedulingOpportunities(priority) > (numNodes * threshold)) { + if (allowed.equals(NodeType.NODE_LOCAL)) { + allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL); + resetSchedulingOpportunities(priority); + } + else if (allowed.equals(NodeType.RACK_LOCAL)) { + allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH); + resetSchedulingOpportunities(priority); + } + } + return allowedLocalityLevel.get(priority); + } + + /** + * Return the level at which we are allowed to schedule containers. + * Given the thresholds indicating how much time passed before relaxing + * scheduling constraints. + */ + public synchronized NodeType getAllowedLocalityLevelByTime(Priority priority, + long nodeLocalityDelayMs, long rackLocalityDelayMs, + long currentTimeMs) { + + // if not being used, can schedule anywhere + if (nodeLocalityDelayMs < 0 || rackLocalityDelayMs < 0) { + return NodeType.OFF_SWITCH; + } + + // default level is NODE_LOCAL + if (! allowedLocalityLevel.containsKey(priority)) { + allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL); + return NodeType.NODE_LOCAL; + } + + NodeType allowed = allowedLocalityLevel.get(priority); + + // if level is already most liberal, we're done + if (allowed.equals(NodeType.OFF_SWITCH)) { + return NodeType.OFF_SWITCH; + } + + // check waiting time + long waitTime = currentTimeMs; + if (lastScheduledContainer.containsKey(priority)) { + waitTime -= lastScheduledContainer.get(priority); + } else { + waitTime -= getStartTime(); + } + + long thresholdTime = allowed.equals(NodeType.NODE_LOCAL) ? + nodeLocalityDelayMs : rackLocalityDelayMs; + + if (waitTime > thresholdTime) { + if (allowed.equals(NodeType.NODE_LOCAL)) { + allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL); + resetSchedulingOpportunities(priority, currentTimeMs); + } else if (allowed.equals(NodeType.RACK_LOCAL)) { + allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH); + resetSchedulingOpportunities(priority, currentTimeMs); + } + } + return allowedLocalityLevel.get(priority); + } + + synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node, + Priority priority, ResourceRequest request, + Container container) { + // Update allowed locality level + NodeType allowed = allowedLocalityLevel.get(priority); + if (allowed != null) { + if (allowed.equals(NodeType.OFF_SWITCH) && + (type.equals(NodeType.NODE_LOCAL) || + type.equals(NodeType.RACK_LOCAL))) { + this.resetAllowedLocalityLevel(priority, type); + } + else if (allowed.equals(NodeType.RACK_LOCAL) && + type.equals(NodeType.NODE_LOCAL)) { + this.resetAllowedLocalityLevel(priority, type); + } + } + + // Required sanity check - AM can call 'allocate' to update resource + // request without locking the scheduler, hence we need to check + if (getTotalRequiredResources(priority) <= 0) { + return null; + } + + // Create RMContainer + RMContainer rmContainer = new RMContainerImpl(container, + getApplicationAttemptId(), node.getNodeID(), + appSchedulingInfo.getUser(), rmContext); + + // Add it to allContainers list. + newlyAllocatedContainers.add(rmContainer); + liveContainers.put(container.getId(), rmContainer); + + // Update consumption and track allocations + List resourceRequestList = appSchedulingInfo.allocate( + type, node, priority, request, container); + Resources.addTo(currentConsumption, container.getResource()); + + // Update resource requests related to "request" and store in RMContainer + ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList); + + // Inform the container + rmContainer.handle( + new RMContainerEvent(container.getId(), RMContainerEventType.START)); + + if (LOG.isDebugEnabled()) { + LOG.debug("allocate: applicationAttemptId=" + + container.getId().getApplicationAttemptId() + + " container=" + container.getId() + " host=" + + container.getNodeId().getHost() + " type=" + type); + } + RMAuditLogger.logSuccess(getUser(), + AuditConstants.ALLOC_CONTAINER, "SchedulerApp", + getApplicationId(), container.getId()); + + return rmContainer; + } + + /** + * Should be called when the scheduler assigns a container at a higher + * degree of locality than the current threshold. Reset the allowed locality + * level to a higher degree of locality. + */ + public synchronized void resetAllowedLocalityLevel(Priority priority, + NodeType level) { + NodeType old = allowedLocalityLevel.get(priority); + LOG.info("Raising locality level from " + old + " to " + level + " at " + + " priority " + priority); + allowedLocalityLevel.put(priority, level); + } + + // related methods + public void addPreemption(RMContainer container, long time) { + assert preemptionMap.get(container) == null; + preemptionMap.put(container, time); + Resources.addTo(preemptedResources, container.getAllocatedResource()); + } + + public Long getContainerPreemptionTime(RMContainer container) { + return preemptionMap.get(container); + } + + public Set getPreemptionContainers() { + return preemptionMap.keySet(); + } + + @Override + public FSLeafQueue getQueue() { + return (FSLeafQueue)super.getQueue(); + } + + public Resource getPreemptedResources() { + return preemptedResources; + } + + public void resetPreemptedResources() { + preemptedResources = Resources.createResource(0); + for (RMContainer container : getPreemptionContainers()) { + Resources.addTo(preemptedResources, container.getAllocatedResource()); + } + } + + public void clearPreemptedResources() { + preemptedResources.setMemory(0); + preemptedResources.setVirtualCores(0); + } + + /** + * Create and return a container object reflecting an allocation for the + * given appliction on the given node with the given capability and + * priority. + */ + public Container createContainer( + FSSchedulerNode node, Resource capability, Priority priority) { + + NodeId nodeId = node.getRMNode().getNodeID(); + ContainerId containerId = BuilderUtils.newContainerId( + getApplicationAttemptId(), getNewContainerId()); + + // Create the container + Container container = + BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() + .getHttpAddress(), capability, priority, null); + + return container; + } + + /** + * Reserve a spot for {@code container} on this {@code node}. If + * the container is {@code alreadyReserved} on the node, simply + * update relevant bookeeping. This dispatches ro relevant handlers + * in {@link FSSchedulerNode}.. + */ + private void reserve(Priority priority, FSSchedulerNode node, + Container container, boolean alreadyReserved) { + LOG.info("Making reservation: node=" + node.getNodeName() + + " app_id=" + getApplicationId()); + + if (!alreadyReserved) { + getMetrics().reserveResource(getUser(), container.getResource()); + RMContainer rmContainer = + super.reserve(node, priority, null, container); + node.reserveResource(this, priority, rmContainer); + } else { + RMContainer rmContainer = node.getReservedContainer(); + super.reserve(node, priority, rmContainer, container); + node.reserveResource(this, priority, rmContainer); + } + } + + /** + * Remove the reservation on {@code node} at the given {@link Priority}. + * This dispatches SchedulerNode handlers as well. + */ + public void unreserve(Priority priority, FSSchedulerNode node) { + RMContainer rmContainer = node.getReservedContainer(); + unreserveInternal(priority, node); + node.unreserveResource(this); + getMetrics().unreserveResource( + getUser(), rmContainer.getContainer().getResource()); + } + + /** + * Assign a container to this node to facilitate {@code request}. If node does + * not have enough memory, create a reservation. This is called once we are + * sure the particular request should be facilitated by this node. + * + * @param node + * The node to try placing the container on. + * @param request + * The ResourceRequest we're trying to satisfy. + * @param type + * The locality of the assignment. + * @param reserved + * Whether there's already a container reserved for this app on the node. + * @return + * If an assignment was made, returns the resources allocated to the + * container. If a reservation was made, returns + * FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was + * made, returns an empty resource. + */ + private Resource assignContainer( + FSSchedulerNode node, ResourceRequest request, NodeType type, + boolean reserved) { + + // How much does this request need? + Resource capability = request.getCapability(); + + // How much does the node have? + Resource available = node.getAvailableResource(); + + Container container = null; + if (reserved) { + container = node.getReservedContainer().getContainer(); + } else { + container = createContainer(node, capability, request.getPriority()); + } + + // Can we allocate a container on this node? + if (Resources.fitsIn(capability, available)) { + // Inform the application of the new container for this request + RMContainer allocatedContainer = + allocate(type, node, request.getPriority(), request, container); + if (allocatedContainer == null) { + // Did the application need this resource? + if (reserved) { + unreserve(request.getPriority(), node); + } + return Resources.none(); + } + + // If we had previously made a reservation, delete it + if (reserved) { + unreserve(request.getPriority(), node); + } + + // Inform the node + node.allocateContainer(allocatedContainer); + + // If this container is used to run AM, update the leaf queue's AM usage + if (getLiveContainers().size() == 1 && !getUnmanagedAM()) { + getQueue().addAMResourceUsage(container.getResource()); + setAmRunning(true); + } + + return container.getResource(); + } else { + // The desired container won't fit here, so reserve + reserve(request.getPriority(), node, container, reserved); + + return FairScheduler.CONTAINER_RESERVED; + } + } + + private Resource assignContainer(FSSchedulerNode node, boolean reserved) { + if (LOG.isDebugEnabled()) { + LOG.debug("Node offered to app: " + getName() + " reserved: " + reserved); + } + + Collection prioritiesToTry = (reserved) ? + Arrays.asList(node.getReservedContainer().getReservedPriority()) : + getPriorities(); + + // For each priority, see if we can schedule a node local, rack local + // or off-switch request. Rack of off-switch requests may be delayed + // (not scheduled) in order to promote better locality. + synchronized (this) { + for (Priority priority : prioritiesToTry) { + if (getTotalRequiredResources(priority) <= 0 || + !hasContainerForNode(priority, node)) { + continue; + } + + addSchedulingOpportunity(priority); + + // Check the AM resource usage for the leaf queue + if (getLiveContainers().size() == 0 && !getUnmanagedAM()) { + if (!getQueue().canRunAppAM(getAMResource())) { + return Resources.none(); + } + } + + ResourceRequest rackLocalRequest = getResourceRequest(priority, + node.getRackName()); + ResourceRequest localRequest = getResourceRequest(priority, + node.getNodeName()); + + if (localRequest != null && !localRequest.getRelaxLocality()) { + LOG.warn("Relax locality off is not supported on local request: " + + localRequest); + } + + NodeType allowedLocality; + if (scheduler.isContinuousSchedulingEnabled()) { + allowedLocality = getAllowedLocalityLevelByTime(priority, + scheduler.getNodeLocalityDelayMs(), + scheduler.getRackLocalityDelayMs(), + scheduler.getClock().getTime()); + } else { + allowedLocality = getAllowedLocalityLevel(priority, + scheduler.getNumClusterNodes(), + scheduler.getNodeLocalityThreshold(), + scheduler.getRackLocalityThreshold()); + } + + if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 + && localRequest != null && localRequest.getNumContainers() != 0) { + return assignContainer(node, localRequest, + NodeType.NODE_LOCAL, reserved); + } + + if (rackLocalRequest != null && !rackLocalRequest.getRelaxLocality()) { + continue; + } + + if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 + && (allowedLocality.equals(NodeType.RACK_LOCAL) || + allowedLocality.equals(NodeType.OFF_SWITCH))) { + return assignContainer(node, rackLocalRequest, + NodeType.RACK_LOCAL, reserved); + } + + ResourceRequest offSwitchRequest = + getResourceRequest(priority, ResourceRequest.ANY); + if (offSwitchRequest != null && !offSwitchRequest.getRelaxLocality()) { + continue; + } + + if (offSwitchRequest != null && offSwitchRequest.getNumContainers() != 0 + && allowedLocality.equals(NodeType.OFF_SWITCH)) { + return assignContainer(node, offSwitchRequest, + NodeType.OFF_SWITCH, reserved); + } + } + } + return Resources.none(); + } + + /** + * Called when this application already has an existing reservation on the + * given node. Sees whether we can turn the reservation into an allocation. + * Also checks whether the application needs the reservation anymore, and + * releases it if not. + * + * @param node + * Node that the application has an existing reservation on + */ + public Resource assignReservedContainer(FSSchedulerNode node) { + RMContainer rmContainer = node.getReservedContainer(); + Priority priority = rmContainer.getReservedPriority(); + + // Make sure the application still needs requests at this priority + if (getTotalRequiredResources(priority) == 0) { + unreserve(priority, node); + return Resources.none(); + } + + // Fail early if the reserved container won't fit. + // Note that we have an assumption here that there's only one container size + // per priority. + if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(), + node.getAvailableResource())) { + return Resources.none(); + } + + return assignContainer(node, true); + } + + + /** + * Whether this app has containers requests that could be satisfied on the + * given node, if the node had full space. + */ + public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) { + ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY); + ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName()); + ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName()); + + return + // There must be outstanding requests at the given priority: + anyRequest != null && anyRequest.getNumContainers() > 0 && + // If locality relaxation is turned off at *-level, there must be a + // non-zero request for the node's rack: + (anyRequest.getRelaxLocality() || + (rackRequest != null && rackRequest.getNumContainers() > 0)) && + // If locality relaxation is turned off at rack-level, there must be a + // non-zero request at the node: + (rackRequest == null || rackRequest.getRelaxLocality() || + (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && + // The requested container must be able to fit on the node: + Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, + anyRequest.getCapability(), node.getRMNode().getTotalCapability()); + } + + + static class RMContainerComparator implements Comparator, + Serializable { + @Override + public int compare(RMContainer c1, RMContainer c2) { + int ret = c1.getContainer().getPriority().compareTo( + c2.getContainer().getPriority()); + if (ret == 0) { + return c2.getContainerId().compareTo(c1.getContainerId()); + } + return ret; + } + } + + /* Schedulable methods implementation */ + + @Override + public String getName() { + return getApplicationId().toString(); + } + + @Override + public Resource getDemand() { + return demand; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public Resource getMinShare() { + return Resources.none(); + } + + @Override + public Resource getMaxShare() { + return Resources.unbounded(); + } + + @Override + public Resource getResourceUsage() { + // Here the getPreemptedResources() always return zero, except in + // a preemption round + return Resources.subtract(getCurrentConsumption(), getPreemptedResources()); + } + + @Override + public ResourceWeights getWeights() { + return scheduler.getAppWeight(this); + } + + @Override + public Priority getPriority() { + // Right now per-app priorities are not passed to scheduler, + // so everyone has the same priority. + return priority; + } + + @Override + public Resource getFairShare() { + return this.fairShare; + } + + @Override + public void setFairShare(Resource fairShare) { + this.fairShare = fairShare; + } + + @Override + public boolean isActive() { + return true; + } + + + @Override + public void updateDemand() { + demand = Resources.createResource(0); + // Demand is current consumption plus outstanding requests + Resources.addTo(demand, getCurrentConsumption()); + + // Add up outstanding resource requests + synchronized (this) { + for (Priority p : getPriorities()) { + for (ResourceRequest r : getResourceRequests(p).values()) { + Resource total = Resources.multiply(r.getCapability(), r.getNumContainers()); + Resources.addTo(demand, total); + } + } + } + } + + @Override + public Resource assignContainer(FSSchedulerNode node) { + return assignContainer(node, false); + } + + /** + * Preempt a running container according to the priority + */ + @Override + public RMContainer preemptContainer() { + if (LOG.isDebugEnabled()) { + LOG.debug("App " + getName() + " is going to preempt a running " + + "container"); + } + + RMContainer toBePreempted = null; + for (RMContainer container : getLiveContainers()) { + if (!getPreemptionContainers().contains(container) && + (toBePreempted == null || + comparator.compare(toBePreempted, container) > 0)) { + toBePreempted = container; + } + } + return toBePreempted; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 3b3f6ce2296..49e8ef06122 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -44,11 +44,11 @@ import org.apache.hadoop.yarn.util.resource.Resources; public class FSLeafQueue extends FSQueue { private static final Log LOG = LogFactory.getLog( FSLeafQueue.class.getName()); - - private final List runnableAppScheds = // apps that are runnable - new ArrayList(); - private final List nonRunnableAppScheds = - new ArrayList(); + + private final List runnableApps = // apps that are runnable + new ArrayList(); + private final List nonRunnableApps = + new ArrayList(); private Resource demand = Resources.createResource(0); @@ -70,33 +70,31 @@ public class FSLeafQueue extends FSQueue { amResourceUsage = Resource.newInstance(0, 0); } - public void addApp(FSSchedulerApp app, boolean runnable) { - AppSchedulable appSchedulable = new AppSchedulable(scheduler, app, this); - app.setAppSchedulable(appSchedulable); + public void addApp(FSAppAttempt app, boolean runnable) { if (runnable) { - runnableAppScheds.add(appSchedulable); + runnableApps.add(app); } else { - nonRunnableAppScheds.add(appSchedulable); + nonRunnableApps.add(app); } } // for testing - void addAppSchedulable(AppSchedulable appSched) { - runnableAppScheds.add(appSched); + void addAppSchedulable(FSAppAttempt appSched) { + runnableApps.add(appSched); } /** * Removes the given app from this queue. * @return whether or not the app was runnable */ - public boolean removeApp(FSSchedulerApp app) { - if (runnableAppScheds.remove(app.getAppSchedulable())) { + public boolean removeApp(FSAppAttempt app) { + if (runnableApps.remove(app)) { // Update AM resource usage if (app.isAmRunning() && app.getAMResource() != null) { Resources.subtractFrom(amResourceUsage, app.getAMResource()); } return true; - } else if (nonRunnableAppScheds.remove(app.getAppSchedulable())) { + } else if (nonRunnableApps.remove(app)) { return false; } else { throw new IllegalStateException("Given app to remove " + app + @@ -104,22 +102,22 @@ public class FSLeafQueue extends FSQueue { } } - public Collection getRunnableAppSchedulables() { - return runnableAppScheds; + public Collection getRunnableAppSchedulables() { + return runnableApps; } - public List getNonRunnableAppSchedulables() { - return nonRunnableAppScheds; + public List getNonRunnableAppSchedulables() { + return nonRunnableApps; } @Override public void collectSchedulerApplications( Collection apps) { - for (AppSchedulable appSched : runnableAppScheds) { - apps.add(appSched.getApp().getApplicationAttemptId()); + for (FSAppAttempt appSched : runnableApps) { + apps.add(appSched.getApplicationAttemptId()); } - for (AppSchedulable appSched : nonRunnableAppScheds) { - apps.add(appSched.getApp().getApplicationAttemptId()); + for (FSAppAttempt appSched : nonRunnableApps) { + apps.add(appSched.getApplicationAttemptId()); } } @@ -145,10 +143,10 @@ public class FSLeafQueue extends FSQueue { @Override public Resource getResourceUsage() { Resource usage = Resources.createResource(0); - for (AppSchedulable app : runnableAppScheds) { + for (FSAppAttempt app : runnableApps) { Resources.addTo(usage, app.getResourceUsage()); } - for (AppSchedulable app : nonRunnableAppScheds) { + for (FSAppAttempt app : nonRunnableApps) { Resources.addTo(usage, app.getResourceUsage()); } return usage; @@ -165,13 +163,13 @@ public class FSLeafQueue extends FSQueue { Resource maxRes = scheduler.getAllocationConfiguration() .getMaxResources(getName()); demand = Resources.createResource(0); - for (AppSchedulable sched : runnableAppScheds) { + for (FSAppAttempt sched : runnableApps) { if (Resources.equals(demand, maxRes)) { break; } updateDemandForApp(sched, maxRes); } - for (AppSchedulable sched : nonRunnableAppScheds) { + for (FSAppAttempt sched : nonRunnableApps) { if (Resources.equals(demand, maxRes)) { break; } @@ -183,7 +181,7 @@ public class FSLeafQueue extends FSQueue { } } - private void updateDemandForApp(AppSchedulable sched, Resource maxRes) { + private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) { sched.updateDemand(); Resource toAdd = sched.getDemand(); if (LOG.isDebugEnabled()) { @@ -207,9 +205,9 @@ public class FSLeafQueue extends FSQueue { } Comparator comparator = policy.getComparator(); - Collections.sort(runnableAppScheds, comparator); - for (AppSchedulable sched : runnableAppScheds) { - if (SchedulerAppUtils.isBlacklisted(sched.getApp(), node, LOG)) { + Collections.sort(runnableApps, comparator); + for (FSAppAttempt sched : runnableApps) { + if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) { continue; } @@ -237,8 +235,8 @@ public class FSLeafQueue extends FSQueue { // Choose the app that is most over fair share Comparator comparator = policy.getComparator(); - AppSchedulable candidateSched = null; - for (AppSchedulable sched : runnableAppScheds) { + FSAppAttempt candidateSched = null; + for (FSAppAttempt sched : runnableApps) { if (candidateSched == null || comparator.compare(sched, candidateSched) > 0) { candidateSched = sched; @@ -291,7 +289,7 @@ public class FSLeafQueue extends FSQueue { @Override public int getNumRunnableApps() { - return runnableAppScheds.size(); + return runnableApps.size(); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 1e94046100a..c071c73321d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -39,7 +39,8 @@ import org.apache.hadoop.yarn.util.resource.Resources; @Private @Unstable -public abstract class FSQueue extends Schedulable implements Queue { +public abstract class FSQueue implements Queue, Schedulable { + private Resource fairShare = Resources.createResource(0, 0); private final String name; protected final FairScheduler scheduler; private final FSQueueMetrics metrics; @@ -139,10 +140,15 @@ public abstract class FSQueue extends Schedulable implements Queue { public FSQueueMetrics getMetrics() { return metrics; } - + + /** Get the fair share assigned to this Schedulable. */ + public Resource getFairShare() { + return fairShare; + } + @Override public void setFairShare(Resource fairShare) { - super.setFairShare(fairShare); + this.fairShare = fairShare; metrics.setFairShare(fairShare); } @@ -187,4 +193,16 @@ public abstract class FSQueue extends Schedulable implements Queue { } return true; } + + @Override + public boolean isActive() { + return getNumRunnableApps() > 0; + } + + /** Convenient toString implementation for debugging. */ + @Override + public String toString() { + return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]", + getName(), getDemand(), getResourceUsage(), fairShare, getWeights()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java index 69f2ab3d844..be08dff397a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java @@ -35,7 +35,7 @@ public class FSSchedulerNode extends SchedulerNode { private static final Log LOG = LogFactory.getLog(FSSchedulerNode.class); - private AppSchedulable reservedAppSchedulable; + private FSAppAttempt reservedAppSchedulable; public FSSchedulerNode(RMNode node, boolean usePortForNodeName) { super(node, usePortForNodeName); @@ -76,7 +76,7 @@ public class FSSchedulerNode extends SchedulerNode { " on node " + this + " for application " + application); } setReservedContainer(container); - this.reservedAppSchedulable = ((FSSchedulerApp) application).getAppSchedulable(); + this.reservedAppSchedulable = (FSAppAttempt) application; } @Override @@ -98,7 +98,7 @@ public class FSSchedulerNode extends SchedulerNode { this.reservedAppSchedulable = null; } - public synchronized AppSchedulable getReservedAppSchedulable() { + public synchronized FSAppAttempt getReservedAppSchedulable() { return reservedAppSchedulable; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 8765ba04dc7..4748d73e038 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -117,7 +117,7 @@ import com.google.common.base.Preconditions; @Unstable @SuppressWarnings("unchecked") public class FairScheduler extends - AbstractYarnScheduler { + AbstractYarnScheduler { private FairSchedulerConfiguration conf; private Resource incrAllocation; @@ -432,8 +432,8 @@ public class FairScheduler extends try { // Reset preemptedResource for each app for (FSLeafQueue queue : getQueueManager().getLeafQueues()) { - for (AppSchedulable app : queue.getRunnableAppSchedulables()) { - app.getApp().resetPreemptedResources(); + for (FSAppAttempt app : queue.getRunnableAppSchedulables()) { + app.resetPreemptedResources(); } } @@ -453,8 +453,8 @@ public class FairScheduler extends } finally { // Clear preemptedResources for each app for (FSLeafQueue queue : getQueueManager().getLeafQueues()) { - for (AppSchedulable app : queue.getRunnableAppSchedulables()) { - app.getApp().clearPreemptedResources(); + for (FSAppAttempt app : queue.getRunnableAppSchedulables()) { + app.clearPreemptedResources(); } } } @@ -465,7 +465,7 @@ public class FairScheduler extends protected void warnOrKillContainer(RMContainer container) { ApplicationAttemptId appAttemptId = container.getApplicationAttemptId(); - FSSchedulerApp app = getSchedulerApp(appAttemptId); + FSAppAttempt app = getSchedulerApp(appAttemptId); FSLeafQueue queue = app.getQueue(); LOG.info("Preempting container (prio=" + container.getContainer().getPriority() + "res=" + container.getContainer().getResource() + @@ -490,7 +490,7 @@ public class FairScheduler extends (getClock().getTime() - time) + "ms)"); } } else { - // track the request in the FSSchedulerApp itself + // track the request in the FSAppAttempt itself app.addPreemption(container, getClock().getTime()); } } @@ -541,7 +541,7 @@ public class FairScheduler extends } // synchronized for sizeBasedWeight - public synchronized ResourceWeights getAppWeight(AppSchedulable app) { + public synchronized ResourceWeights getAppWeight(FSAppAttempt app) { double weight = 1.0; if (sizeBasedWeight) { // Set weight based on current memory demand @@ -636,8 +636,8 @@ public class FairScheduler extends return; } - SchedulerApplication application = - new SchedulerApplication(queue, user); + SchedulerApplication application = + new SchedulerApplication(queue, user); applications.put(applicationId, application); queue.getMetrics().submitApp(user); @@ -661,13 +661,13 @@ public class FairScheduler extends ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering) { - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationAttemptId.getApplicationId()); String user = application.getUser(); FSLeafQueue queue = (FSLeafQueue) application.getQueue(); - FSSchedulerApp attempt = - new FSSchedulerApp(applicationAttemptId, user, + FSAppAttempt attempt = + new FSAppAttempt(this, applicationAttemptId, user, queue, new ActiveUsersManager(getRootQueueMetrics()), rmContext); if (transferStateFromPreviousAttempt) { @@ -742,7 +742,7 @@ public class FairScheduler extends private synchronized void removeApplication(ApplicationId applicationId, RMAppState finalState) { - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationId); if (application == null){ LOG.warn("Couldn't find application " + applicationId); @@ -757,9 +757,9 @@ public class FairScheduler extends RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { LOG.info("Application " + applicationAttemptId + " is done." + " finalState=" + rmAppAttemptFinalState); - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationAttemptId.getApplicationId()); - FSSchedulerApp attempt = getSchedulerApp(applicationAttemptId); + FSAppAttempt attempt = getSchedulerApp(applicationAttemptId); if (attempt == null || application == null) { LOG.info("Unknown application " + applicationAttemptId + " has completed!"); @@ -820,7 +820,7 @@ public class FairScheduler extends Container container = rmContainer.getContainer(); // Get the application for the finished container - FSSchedulerApp application = + FSAppAttempt application = getCurrentAttemptForContainer(container.getId()); ApplicationId appId = container.getId().getApplicationAttemptId().getApplicationId(); @@ -835,8 +835,7 @@ public class FairScheduler extends FSSchedulerNode node = getFSSchedulerNode(container.getNodeId()); if (rmContainer.getState() == RMContainerState.RESERVED) { - application.unreserve(node, rmContainer.getReservedPriority()); - node.unreserveResource(application); + application.unreserve(rmContainer.getReservedPriority(), node); } else { application.containerCompleted(rmContainer, containerStatus, event); node.releaseContainer(container); @@ -896,7 +895,7 @@ public class FairScheduler extends List ask, List release, List blacklistAdditions, List blacklistRemovals) { // Make sure this application exists - FSSchedulerApp application = getSchedulerApp(appAttemptId); + FSAppAttempt application = getSchedulerApp(appAttemptId); if (application == null) { LOG.info("Calling allocate on removed " + "or non existant application " + appAttemptId); @@ -1066,13 +1065,13 @@ public class FairScheduler extends // 1. Check for reserved applications // 2. Schedule if there are no reservations - AppSchedulable reservedAppSchedulable = node.getReservedAppSchedulable(); + FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable(); if (reservedAppSchedulable != null) { Priority reservedPriority = node.getReservedContainer().getReservedPriority(); if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node)) { // Don't hold the reservation if app can no longer use it LOG.info("Releasing reservation that cannot be satisfied for application " - + reservedAppSchedulable.getApp().getApplicationAttemptId() + + reservedAppSchedulable.getApplicationAttemptId() + " on node " + node); reservedAppSchedulable.unreserve(reservedPriority, node); reservedAppSchedulable = null; @@ -1080,7 +1079,7 @@ public class FairScheduler extends // Reservation exists; try to fulfill the reservation if (LOG.isDebugEnabled()) { LOG.debug("Trying to fulfill reservation for application " - + reservedAppSchedulable.getApp().getApplicationAttemptId() + + reservedAppSchedulable.getApplicationAttemptId() + " on node: " + node); } @@ -1105,8 +1104,8 @@ public class FairScheduler extends updateRootQueueMetrics(); } - public FSSchedulerApp getSchedulerApp(ApplicationAttemptId appAttemptId) { - return (FSSchedulerApp) super.getApplicationAttempt(appAttemptId); + public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) { + return super.getApplicationAttempt(appAttemptId); } /** @@ -1268,8 +1267,8 @@ public class FairScheduler extends fsOpDurations = FSOpDurations.getInstance(true); // This stores per-application scheduling information - this.applications = - new ConcurrentHashMap>(); + this.applications = new ConcurrentHashMap< + ApplicationId, SchedulerApplication>(); this.eventLog = new FairSchedulerEventLog(); eventLog.init(this.conf); @@ -1369,7 +1368,7 @@ public class FairScheduler extends @Override public List getQueueUserAclInfo() { - UserGroupInformation user = null; + UserGroupInformation user; try { user = UserGroupInformation.getCurrentUser(); } catch (IOException ioe) { @@ -1431,11 +1430,11 @@ public class FairScheduler extends @Override public synchronized String moveApplication(ApplicationId appId, String queueName) throws YarnException { - SchedulerApplication app = applications.get(appId); + SchedulerApplication app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); } - FSSchedulerApp attempt = (FSSchedulerApp) app.getCurrentAppAttempt(); + FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt synchronized (attempt) { FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); @@ -1448,8 +1447,7 @@ public class FairScheduler extends return oldQueue.getQueueName(); } - if (oldQueue.getRunnableAppSchedulables().contains( - attempt.getAppSchedulable())) { + if (oldQueue.getRunnableAppSchedulables().contains(attempt)) { verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); } @@ -1458,7 +1456,7 @@ public class FairScheduler extends } } - private void verifyMoveDoesNotViolateConstraints(FSSchedulerApp app, + private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app, FSLeafQueue oldQueue, FSLeafQueue targetQueue) throws YarnException { String queueName = targetQueue.getQueueName(); ApplicationAttemptId appAttId = app.getApplicationAttemptId(); @@ -1495,8 +1493,8 @@ public class FairScheduler extends * Helper for moveApplication, which has appropriate synchronization, so all * operations will be atomic. */ - private void executeMove(SchedulerApplication app, - FSSchedulerApp attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) { + private void executeMove(SchedulerApplication app, + FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) { boolean wasRunnable = oldQueue.removeApp(attempt); // if app was not runnable before, it may be runnable now boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java index c87f4ca6b8c..1b75740a155 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java @@ -25,15 +25,15 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; /** - * Order {@link AppSchedulable} objects by priority and then by submit time, as + * Order {@link FSAppAttempt} objects by priority and then by submit time, as * in the default scheduler in Hadoop. */ @Private @Unstable -public class FifoAppComparator implements Comparator, Serializable { +public class FifoAppComparator implements Comparator, Serializable { private static final long serialVersionUID = 3428835083489547918L; - public int compare(AppSchedulable a1, AppSchedulable a2) { + public int compare(FSAppAttempt a1, FSAppAttempt a2) { int res = a1.getPriority().compareTo(a2.getPriority()); if (res == 0) { if (a1.getStartTime() < a2.getStartTime()) { @@ -44,7 +44,7 @@ public class FifoAppComparator implements Comparator, Serializab } if (res == 0) { // If there is a tie, break it by app ID to get a deterministic order - res = a1.getApp().getApplicationId().compareTo(a2.getApp().getApplicationId()); + res = a1.getApplicationId().compareTo(a2.getApplicationId()); } return res; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java index 359519a2f29..feeda1e90c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java @@ -43,7 +43,7 @@ public class MaxRunningAppsEnforcer { // Tracks the number of running applications by user. private final Map usersNumRunnableApps; @VisibleForTesting - final ListMultimap usersNonRunnableApps; + final ListMultimap usersNonRunnableApps; public MaxRunningAppsEnforcer(FairScheduler scheduler) { this.scheduler = scheduler; @@ -80,7 +80,7 @@ public class MaxRunningAppsEnforcer { * Tracks the given new runnable app for purposes of maintaining max running * app limits. */ - public void trackRunnableApp(FSSchedulerApp app) { + public void trackRunnableApp(FSAppAttempt app) { String user = app.getUser(); FSLeafQueue queue = app.getQueue(); // Increment running counts for all parent queues @@ -99,9 +99,9 @@ public class MaxRunningAppsEnforcer { * Tracks the given new non runnable app so that it can be made runnable when * it would not violate max running app limits. */ - public void trackNonRunnableApp(FSSchedulerApp app) { + public void trackNonRunnableApp(FSAppAttempt app) { String user = app.getUser(); - usersNonRunnableApps.put(user, app.getAppSchedulable()); + usersNonRunnableApps.put(user, app); } /** @@ -111,7 +111,7 @@ public class MaxRunningAppsEnforcer { * Runs in O(n log(n)) where n is the number of queues that are under the * highest queue that went from having no slack to having slack. */ - public void updateRunnabilityOnAppRemoval(FSSchedulerApp app, FSLeafQueue queue) { + public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); // childqueueX might have no pending apps itself, but if a queue higher up @@ -133,8 +133,8 @@ public class MaxRunningAppsEnforcer { parent = parent.getParent(); } - List> appsNowMaybeRunnable = - new ArrayList>(); + List> appsNowMaybeRunnable = + new ArrayList>(); // Compile lists of apps which may now be runnable // We gather lists instead of building a set of all non-runnable apps so @@ -150,26 +150,26 @@ public class MaxRunningAppsEnforcer { userNumRunning = 0; } if (userNumRunning == allocConf.getUserMaxApps(user) - 1) { - List userWaitingApps = usersNonRunnableApps.get(user); + List userWaitingApps = usersNonRunnableApps.get(user); if (userWaitingApps != null) { appsNowMaybeRunnable.add(userWaitingApps); } } // Scan through and check whether this means that any apps are now runnable - Iterator iter = new MultiListStartTimeIterator( + Iterator iter = new MultiListStartTimeIterator( appsNowMaybeRunnable); - FSSchedulerApp prev = null; - List noLongerPendingApps = new ArrayList(); + FSAppAttempt prev = null; + List noLongerPendingApps = new ArrayList(); while (iter.hasNext()) { - FSSchedulerApp next = iter.next(); + FSAppAttempt next = iter.next(); if (next == prev) { continue; } if (canAppBeRunnable(next.getQueue(), next.getUser())) { trackRunnableApp(next); - AppSchedulable appSched = next.getAppSchedulable(); + FSAppAttempt appSched = next; next.getQueue().getRunnableAppSchedulables().add(appSched); noLongerPendingApps.add(appSched); @@ -186,14 +186,14 @@ public class MaxRunningAppsEnforcer { // We remove the apps from their pending lists afterwards so that we don't // pull them out from under the iterator. If they are not in these lists // in the first place, there is a bug. - for (AppSchedulable appSched : noLongerPendingApps) { - if (!appSched.getApp().getQueue().getNonRunnableAppSchedulables() + for (FSAppAttempt appSched : noLongerPendingApps) { + if (!appSched.getQueue().getNonRunnableAppSchedulables() .remove(appSched)) { LOG.error("Can't make app runnable that does not already exist in queue" + " as non-runnable: " + appSched + ". This should never happen."); } - if (!usersNonRunnableApps.remove(appSched.getApp().getUser(), appSched)) { + if (!usersNonRunnableApps.remove(appSched.getUser(), appSched)) { LOG.error("Waiting app " + appSched + " expected to be in " + "usersNonRunnableApps, but was not. This should never happen."); } @@ -204,7 +204,7 @@ public class MaxRunningAppsEnforcer { * Updates the relevant tracking variables after a runnable app with the given * queue and user has been removed. */ - public void untrackRunnableApp(FSSchedulerApp app) { + public void untrackRunnableApp(FSAppAttempt app) { // Update usersRunnableApps String user = app.getUser(); int newUserNumRunning = usersNumRunnableApps.get(user) - 1; @@ -226,8 +226,8 @@ public class MaxRunningAppsEnforcer { /** * Stops tracking the given non-runnable app */ - public void untrackNonRunnableApp(FSSchedulerApp app) { - usersNonRunnableApps.remove(app.getUser(), app.getAppSchedulable()); + public void untrackNonRunnableApp(FSAppAttempt app) { + usersNonRunnableApps.remove(app.getUser(), app); } /** @@ -235,7 +235,7 @@ public class MaxRunningAppsEnforcer { * of non-runnable applications. */ private void gatherPossiblyRunnableAppLists(FSQueue queue, - List> appLists) { + List> appLists) { if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration() .getQueueMaxApps(queue.getName())) { if (queue instanceof FSLeafQueue) { @@ -259,14 +259,14 @@ public class MaxRunningAppsEnforcer { * of O(num lists) time. */ static class MultiListStartTimeIterator implements - Iterator { + Iterator { - private List[] appLists; + private List[] appLists; private int[] curPositionsInAppLists; private PriorityQueue appListsByCurStartTime; @SuppressWarnings("unchecked") - public MultiListStartTimeIterator(List> appListList) { + public MultiListStartTimeIterator(List> appListList) { appLists = appListList.toArray(new List[appListList.size()]); curPositionsInAppLists = new int[appLists.length]; appListsByCurStartTime = new PriorityQueue(); @@ -284,10 +284,10 @@ public class MaxRunningAppsEnforcer { } @Override - public FSSchedulerApp next() { + public FSAppAttempt next() { IndexAndTime indexAndTime = appListsByCurStartTime.remove(); int nextListIndex = indexAndTime.index; - AppSchedulable next = appLists[nextListIndex] + FSAppAttempt next = appLists[nextListIndex] .get(curPositionsInAppLists[nextListIndex]); curPositionsInAppLists[nextListIndex]++; @@ -299,7 +299,7 @@ public class MaxRunningAppsEnforcer { } appListsByCurStartTime.add(indexAndTime); - return next.getApp(); + return next; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java index e77eed79568..fb32e565808 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java @@ -48,7 +48,7 @@ public class NewAppWeightBooster extends Configured implements WeightAdjuster { super.setConf(conf); } - public double adjustWeight(AppSchedulable app, double curWeight) { + public double adjustWeight(FSAppAttempt app, double curWeight) { long start = app.getStartTime(); long now = System.currentTimeMillis(); if (now - start < duration) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java index 5134be4dce9..122b986defc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java @@ -27,20 +27,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.resource.Resources; /** - * A Schedulable represents an entity that can launch tasks, such as a job - * or a queue. It provides a common interface so that algorithms such as fair - * sharing can be applied both within a queue and across queues. There are - * currently two types of Schedulables: JobSchedulables, which represent a - * single job, and QueueSchedulables, which allocate among jobs in their queue. - * - * Separate sets of Schedulables are used for maps and reduces. Each queue has - * both a mapSchedulable and a reduceSchedulable, and so does each job. + * A Schedulable represents an entity that can be scheduled such as an + * application or a queue. It provides a common interface so that algorithms + * such as fair sharing can be applied both within a queue and across queues. * * A Schedulable is responsible for three roles: - * 1) It can launch tasks through assignTask(). - * 2) It provides information about the job/queue to the scheduler, including: + * 1) Assign resources through {@link #assignContainer}. + * 2) It provides information about the app/queue to the scheduler, including: * - Demand (maximum number of tasks required) - * - Number of currently running tasks * - Minimum share (for queues) * - Job/queue weight (for fair sharing) * - Start time and priority (for FIFO) @@ -57,81 +51,61 @@ import org.apache.hadoop.yarn.util.resource.Resources; */ @Private @Unstable -public abstract class Schedulable { - /** Fair share assigned to this Schedulable */ - private Resource fairShare = Resources.createResource(0); - +public interface Schedulable { /** * Name of job/queue, used for debugging as well as for breaking ties in * scheduling order deterministically. */ - public abstract String getName(); + public String getName(); /** * Maximum number of resources required by this Schedulable. This is defined as * number of currently utilized resources + number of unlaunched resources (that * are either not yet launched or need to be speculated). */ - public abstract Resource getDemand(); + public Resource getDemand(); /** Get the aggregate amount of resources consumed by the schedulable. */ - public abstract Resource getResourceUsage(); + public Resource getResourceUsage(); /** Minimum Resource share assigned to the schedulable. */ - public abstract Resource getMinShare(); + public Resource getMinShare(); /** Maximum Resource share assigned to the schedulable. */ - public abstract Resource getMaxShare(); + public Resource getMaxShare(); /** Job/queue weight in fair sharing. */ - public abstract ResourceWeights getWeights(); + public ResourceWeights getWeights(); /** Start time for jobs in FIFO queues; meaningless for QueueSchedulables.*/ - public abstract long getStartTime(); + public long getStartTime(); /** Job priority for jobs in FIFO queues; meaningless for QueueSchedulables. */ - public abstract Priority getPriority(); + public Priority getPriority(); /** Refresh the Schedulable's demand and those of its children if any. */ - public abstract void updateDemand(); + public void updateDemand(); /** * Assign a container on this node if possible, and return the amount of * resources assigned. */ - public abstract Resource assignContainer(FSSchedulerNode node); + public Resource assignContainer(FSSchedulerNode node); /** * Preempt a container from this Schedulable if possible. */ - public abstract RMContainer preemptContainer(); - - /** Assign a fair share to this Schedulable. */ - public void setFairShare(Resource fairShare) { - this.fairShare = fairShare; - } + public RMContainer preemptContainer(); /** Get the fair share assigned to this Schedulable. */ - public Resource getFairShare() { - return fairShare; - } + public Resource getFairShare(); + + /** Assign a fair share to this Schedulable. */ + public void setFairShare(Resource fairShare); /** * Returns true if queue has atleast one app running. Always returns true for * AppSchedulables. */ - public boolean isActive() { - if (this instanceof FSQueue) { - FSQueue queue = (FSQueue) this; - return queue.getNumRunnableApps() > 0; - } - return true; - } - - /** Convenient toString implementation for debugging. */ - @Override - public String toString() { - return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]", - getName(), getDemand(), getResourceUsage(), fairShare, getWeights()); - } + public boolean isActive(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java index 1a9467fc003..67364ed6e91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java @@ -32,5 +32,5 @@ import org.apache.hadoop.conf.Configurable; @Private @Unstable public interface WeightAdjuster { - public double adjustWeight(AppSchedulable app, double curWeight); + public double adjustWeight(FSAppAttempt app, double curWeight); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java index f136f940107..23f8c01c38a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java @@ -46,8 +46,7 @@ public class FairSchedulerInfo extends SchedulerInfo { } public int getAppFairShare(ApplicationAttemptId appAttemptId) { - return scheduler.getSchedulerApp(appAttemptId). - getAppSchedulable().getFairShare().getMemory(); + return scheduler.getSchedulerApp(appAttemptId).getFairShare().getMemory(); } public FairSchedulerQueueInfo getRootQueueInfo() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java index 5cdfb2abdf7..d389b9f0765 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java @@ -24,7 +24,8 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .FSAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; @@ -39,9 +40,9 @@ public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo { public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) { super(queue, scheduler); - Collection apps = queue.getRunnableAppSchedulables(); - for (AppSchedulable app : apps) { - if (app.getApp().isPending()) { + Collection apps = queue.getRunnableAppSchedulables(); + for (FSAppAttempt app : apps) { + if (app.isPending()) { numPendingApps++; } else { numActiveApps++; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java index dcfc2d3aa2f..5bd52ab7a07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java @@ -28,10 +28,11 @@ import org.apache.hadoop.yarn.util.resource.Resources; /** * Dummy implementation of Schedulable for unit testing. */ -public class FakeSchedulable extends Schedulable { +public class FakeSchedulable implements Schedulable { private Resource usage; private Resource minShare; private Resource maxShare; + private Resource fairShare; private ResourceWeights weights; private Priority priority; private long startTime; @@ -89,6 +90,21 @@ public class FakeSchedulable extends Schedulable { return null; } + @Override + public Resource getFairShare() { + return this.fairShare; + } + + @Override + public void setFairShare(Resource fairShare) { + this.fairShare = fairShare; + } + + @Override + public boolean isActive() { + return true; + } + @Override public Resource getDemand() { return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java new file mode 100644 index 00000000000..0ab1f70147b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; +import org.apache.hadoop.yarn.util.Clock; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestFSAppAttempt extends FairSchedulerTestBase { + + private class MockClock implements Clock { + private long time = 0; + @Override + public long getTime() { + return time; + } + + public void tick(int seconds) { + time = time + seconds * 1000; + } + + } + + @Before + public void setup() { + Configuration conf = createConfiguration(); + resourceManager = new MockRM(conf); + resourceManager.start(); + scheduler = (FairScheduler) resourceManager.getResourceScheduler(); + } + + @Test + public void testDelayScheduling() { + FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); + Priority prio = Mockito.mock(Priority.class); + Mockito.when(prio.getPriority()).thenReturn(1); + double nodeLocalityThreshold = .5; + double rackLocalityThreshold = .6; + + ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); + RMContext rmContext = resourceManager.getRMContext(); + FSAppAttempt schedulerApp = + new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue , + null, rmContext); + + // Default level should be node-local + assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + + // First five scheduling opportunities should remain node local + for (int i = 0; i < 5; i++) { + schedulerApp.addSchedulingOpportunity(prio); + assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + } + + // After five it should switch to rack local + schedulerApp.addSchedulingOpportunity(prio); + assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + + // Manually set back to node local + schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL); + schedulerApp.resetSchedulingOpportunities(prio); + assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + + // Now escalate again to rack-local, then to off-switch + for (int i = 0; i < 5; i++) { + schedulerApp.addSchedulingOpportunity(prio); + assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + } + + schedulerApp.addSchedulingOpportunity(prio); + assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + + for (int i = 0; i < 6; i++) { + schedulerApp.addSchedulingOpportunity(prio); + assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + } + + schedulerApp.addSchedulingOpportunity(prio); + assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel( + prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); + } + + @Test + public void testDelaySchedulingForContinuousScheduling() + throws InterruptedException { + FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue("queue", true); + Priority prio = Mockito.mock(Priority.class); + Mockito.when(prio.getPriority()).thenReturn(1); + + MockClock clock = new MockClock(); + scheduler.setClock(clock); + + long nodeLocalityDelayMs = 5 * 1000L; // 5 seconds + long rackLocalityDelayMs = 6 * 1000L; // 6 seconds + + RMContext rmContext = resourceManager.getRMContext(); + ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); + FSAppAttempt schedulerApp = + new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, + null, rmContext); + + // Default level should be node-local + assertEquals(NodeType.NODE_LOCAL, + schedulerApp.getAllowedLocalityLevelByTime(prio, + nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); + + // after 4 seconds should remain node local + clock.tick(4); + assertEquals(NodeType.NODE_LOCAL, + schedulerApp.getAllowedLocalityLevelByTime(prio, + nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); + + // after 6 seconds should switch to rack local + clock.tick(2); + assertEquals(NodeType.RACK_LOCAL, + schedulerApp.getAllowedLocalityLevelByTime(prio, + nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); + + // manually set back to node local + schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL); + schedulerApp.resetSchedulingOpportunities(prio, clock.getTime()); + assertEquals(NodeType.NODE_LOCAL, + schedulerApp.getAllowedLocalityLevelByTime(prio, + nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); + + // Now escalate again to rack-local, then to off-switch + clock.tick(6); + assertEquals(NodeType.RACK_LOCAL, + schedulerApp.getAllowedLocalityLevelByTime(prio, + nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); + + clock.tick(7); + assertEquals(NodeType.OFF_SWITCH, + schedulerApp.getAllowedLocalityLevelByTime(prio, + nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); + } + + @Test + /** + * Ensure that when negative paramaters are given (signaling delay scheduling + * no tin use), the least restrictive locality level is returned. + */ + public void testLocalityLevelWithoutDelays() { + FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); + Priority prio = Mockito.mock(Priority.class); + Mockito.when(prio.getPriority()).thenReturn(1); + + RMContext rmContext = resourceManager.getRMContext(); + ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); + FSAppAttempt schedulerApp = + new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue , + null, rmContext); + assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel( + prio, 10, -1.0, -1.0)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index 4c07ee885f6..7323b6ab050 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -62,7 +62,7 @@ public class TestFSLeafQueue { @Test public void testUpdateDemand() { - AppSchedulable app = mock(AppSchedulable.class); + FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(maxResource); schedulable.addAppSchedulable(app); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0ada021aa20..a7b1738cda7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; @@ -82,13 +81,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -1539,7 +1536,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); - FSSchedulerApp app = scheduler.getSchedulerApp(attId); + FSAppAttempt app = scheduler.getSchedulerApp(attId); assertEquals(1, app.getLiveContainers().size()); ContainerId containerId = scheduler.getSchedulerApp(attId) @@ -1613,9 +1610,9 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1", "norealuserhasthisname2", 1); - FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); assertNotNull("The application was not allowed", app1); - FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); assertNull("The application was allowed", app2); } @@ -1688,8 +1685,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { "user1", 2); ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1", "user1", 2); - FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1); - FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true); queue1.setPolicy(new FifoPolicy()); @@ -1731,7 +1728,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId = createSchedulingRequest(1024, "root.default", "user", 8); - FSSchedulerApp app = scheduler.getSchedulerApp(attId); + FSAppAttempt app = scheduler.getSchedulerApp(attId); // set maxAssign to 2: only 2 containers should be allocated scheduler.maxAssign = 2; @@ -1766,7 +1763,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId = createSchedulingRequest(0, 1, "root.default", "user", 8); - FSSchedulerApp app = scheduler.getSchedulerApp(attId); + FSAppAttempt app = scheduler.getSchedulerApp(attId); // set maxAssign to 2: only 2 containers should be allocated scheduler.maxAssign = 2; @@ -1830,10 +1827,10 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId4 = createSchedulingRequest(1024, fifoQueue, user, 4); - FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1); - FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2); - FSSchedulerApp app3 = scheduler.getSchedulerApp(attId3); - FSSchedulerApp app4 = scheduler.getSchedulerApp(attId4); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); + FSAppAttempt app3 = scheduler.getSchedulerApp(attId3); + FSAppAttempt app4 = scheduler.getSchedulerApp(attId4); scheduler.getQueueManager().getLeafQueue(fifoQueue, true) .setPolicy(SchedulingPolicy.parse("fifo")); @@ -1952,7 +1949,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); - FSSchedulerApp app = scheduler.getSchedulerApp(attId); + FSAppAttempt app = scheduler.getSchedulerApp(attId); assertEquals(0, app.getLiveContainers().size()); assertEquals(0, app.getReservedContainers().size()); @@ -2025,7 +2022,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { NodeUpdateSchedulerEvent node2UpdateEvent = new NodeUpdateSchedulerEvent(node2); // no matter how many heartbeats, node2 should never get a container - FSSchedulerApp app = scheduler.getSchedulerApp(attId1); + FSAppAttempt app = scheduler.getSchedulerApp(attId1); for (int i = 0; i < 10; i++) { scheduler.handle(node2UpdateEvent); assertEquals(0, app.getLiveContainers().size()); @@ -2066,7 +2063,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { NodeUpdateSchedulerEvent node2UpdateEvent = new NodeUpdateSchedulerEvent(node2); // no matter how many heartbeats, node2 should never get a container - FSSchedulerApp app = scheduler.getSchedulerApp(attId1); + FSAppAttempt app = scheduler.getSchedulerApp(attId1); for (int i = 0; i < 10; i++) { scheduler.handle(node2UpdateEvent); assertEquals(0, app.getLiveContainers().size()); @@ -2101,7 +2098,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId = createSchedulingRequest(1024, "queue1", "user1", 0); - FSSchedulerApp app = scheduler.getSchedulerApp(attId); + FSAppAttempt app = scheduler.getSchedulerApp(attId); ResourceRequest nodeRequest = createResourceRequest(1024, node2.getHostName(), 1, 2, true); ResourceRequest rackRequest = createResourceRequest(1024, "rack1", 1, 2, true); @@ -2143,7 +2140,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); - FSSchedulerApp app = scheduler.getSchedulerApp(attId); + FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); @@ -2165,10 +2162,10 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId appAttId1 = createSchedulingRequest(2048, 1, "queue1", "user1", 2); - FSSchedulerApp app1 = scheduler.getSchedulerApp(appAttId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(appAttId1); ApplicationAttemptId appAttId2 = createSchedulingRequest(1024, 2, "queue1", "user1", 2); - FSSchedulerApp app2 = scheduler.getSchedulerApp(appAttId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2); DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy(); drfPolicy.initialize(scheduler.getClusterResource()); @@ -2208,13 +2205,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId appAttId1 = createSchedulingRequest(3072, 1, "queue1", "user1", 2); - FSSchedulerApp app1 = scheduler.getSchedulerApp(appAttId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(appAttId1); ApplicationAttemptId appAttId2 = createSchedulingRequest(2048, 2, "queue1", "user1", 2); - FSSchedulerApp app2 = scheduler.getSchedulerApp(appAttId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2); ApplicationAttemptId appAttId3 = createSchedulingRequest(1024, 2, "queue2", "user1", 2); - FSSchedulerApp app3 = scheduler.getSchedulerApp(appAttId3); + FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3); DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy(); drfPolicy.initialize(scheduler.getClusterResource()); @@ -2247,19 +2244,19 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId appAttId1 = createSchedulingRequest(3074, 1, "queue1.subqueue1", "user1", 2); Thread.sleep(3); // so that start times will be different - FSSchedulerApp app1 = scheduler.getSchedulerApp(appAttId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(appAttId1); ApplicationAttemptId appAttId2 = createSchedulingRequest(1024, 3, "queue1.subqueue1", "user1", 2); Thread.sleep(3); // so that start times will be different - FSSchedulerApp app2 = scheduler.getSchedulerApp(appAttId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2); ApplicationAttemptId appAttId3 = createSchedulingRequest(2048, 2, "queue1.subqueue2", "user1", 2); Thread.sleep(3); // so that start times will be different - FSSchedulerApp app3 = scheduler.getSchedulerApp(appAttId3); + FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3); ApplicationAttemptId appAttId4 = createSchedulingRequest(1024, 2, "queue2", "user1", 2); Thread.sleep(3); // so that start times will be different - FSSchedulerApp app4 = scheduler.getSchedulerApp(appAttId4); + FSAppAttempt app4 = scheduler.getSchedulerApp(appAttId4); DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy(); drfPolicy.initialize(scheduler.getClusterResource()); @@ -2341,7 +2338,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { NodeUpdateSchedulerEvent(node2); // no matter how many heartbeats, node2 should never get a container - FSSchedulerApp app = scheduler.getSchedulerApp(attId1); + FSAppAttempt app = scheduler.getSchedulerApp(attId1); for (int i = 0; i < 10; i++) { scheduler.handle(node2UpdateEvent); assertEquals(0, app.getLiveContainers().size()); @@ -2353,14 +2350,14 @@ public class TestFairScheduler extends FairSchedulerTestBase { } private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) { - FSSchedulerApp app = scheduler.getSchedulerApp(attId); + FSAppAttempt app = scheduler.getSchedulerApp(attId); FSLeafQueue queue = app.getQueue(); - Collection runnableApps = + Collection runnableApps = queue.getRunnableAppSchedulables(); - Collection nonRunnableApps = + Collection nonRunnableApps = queue.getNonRunnableAppSchedulables(); - assertEquals(runnable, runnableApps.contains(app.getAppSchedulable())); - assertEquals(!runnable, nonRunnableApps.contains(app.getAppSchedulable())); + assertEquals(runnable, runnableApps.contains(app)); + assertEquals(!runnable, nonRunnableApps.contains(app)); } private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue, @@ -2465,7 +2462,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId1 = createAppAttemptId(1, 1); createApplicationWithAMResource(attId1, "queue1", "user1", amResource1); createSchedulingRequestExistingApplication(1024, 1, amPriority, attId1); - FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application1's AM requests 1024 MB memory", @@ -2479,7 +2476,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId2 = createAppAttemptId(2, 1); createApplicationWithAMResource(attId2, "queue1", "user1", amResource1); createSchedulingRequestExistingApplication(1024, 1, amPriority, attId2); - FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application2's AM requests 1024 MB memory", @@ -2493,7 +2490,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId3 = createAppAttemptId(3, 1); createApplicationWithAMResource(attId3, "queue1", "user1", amResource1); createSchedulingRequestExistingApplication(1024, 1, amPriority, attId3); - FSSchedulerApp app3 = scheduler.getSchedulerApp(attId3); + FSAppAttempt app3 = scheduler.getSchedulerApp(attId3); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application3's AM requests 1024 MB memory", @@ -2529,7 +2526,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId4 = createAppAttemptId(4, 1); createApplicationWithAMResource(attId4, "queue1", "user1", amResource2); createSchedulingRequestExistingApplication(2048, 2, amPriority, attId4); - FSSchedulerApp app4 = scheduler.getSchedulerApp(attId4); + FSAppAttempt app4 = scheduler.getSchedulerApp(attId4); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application4's AM requests 2048 MB memory", @@ -2543,7 +2540,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId5 = createAppAttemptId(5, 1); createApplicationWithAMResource(attId5, "queue1", "user1", amResource2); createSchedulingRequestExistingApplication(2048, 2, amPriority, attId5); - FSSchedulerApp app5 = scheduler.getSchedulerApp(attId5); + FSAppAttempt app5 = scheduler.getSchedulerApp(attId5); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application5's AM requests 2048 MB memory", @@ -2586,7 +2583,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId6 = createAppAttemptId(6, 1); createApplicationWithAMResource(attId6, "queue1", "user1", amResource3); createSchedulingRequestExistingApplication(1860, 2, amPriority, attId6); - FSSchedulerApp app6 = scheduler.getSchedulerApp(attId6); + FSAppAttempt app6 = scheduler.getSchedulerApp(attId6); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application6's AM should not be running", @@ -2677,7 +2674,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId1 = createAppAttemptId(1, 1); createApplicationWithAMResource(attId1, "queue1", "test1", amResource1); createSchedulingRequestExistingApplication(2048, 1, amPriority, attId1); - FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application1's AM requests 2048 MB memory", @@ -2691,7 +2688,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId2 = createAppAttemptId(2, 1); createApplicationWithAMResource(attId2, "queue2", "test1", amResource1); createSchedulingRequestExistingApplication(2048, 1, amPriority, attId2); - FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Application2's AM requests 2048 MB memory", @@ -2823,7 +2820,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { // at least one pass Thread.sleep(fs.getConf().getContinuousSchedulingSleepMs() + 500); - FSSchedulerApp app = fs.getSchedulerApp(appAttemptId); + FSAppAttempt app = fs.getSchedulerApp(appAttemptId); // Wait until app gets resources. while (app.getCurrentConsumption().equals(Resources.none())) { } @@ -3007,7 +3004,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(1, scheduler.getSchedulerApp(appAttemptId).getLiveContainers() .size()); - FSSchedulerApp app = scheduler.getSchedulerApp(appAttemptId); + FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId); // ResourceRequest will be empty once NodeUpdate is completed Assert.assertNull(app.getResourceRequest(priority, host)); @@ -3063,7 +3060,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId appAttemptId = createSchedulingRequest(GB, "root.default", "user", 1); - FSSchedulerApp app = scheduler.getSchedulerApp(appAttemptId); + FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId); // Verify the blacklist can be updated independent of requesting containers scheduler.allocate(appAttemptId, Collections.emptyList(), @@ -3171,12 +3168,10 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand()); scheduler.moveApplication(appId, "queue2"); - FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + FSAppAttempt app = scheduler.getSchedulerApp(appAttId); assertSame(targetQueue, app.getQueue()); - assertFalse(oldQueue.getRunnableAppSchedulables() - .contains(app.getAppSchedulable())); - assertTrue(targetQueue.getRunnableAppSchedulables() - .contains(app.getAppSchedulable())); + assertFalse(oldQueue.getRunnableAppSchedulables().contains(app)); + assertTrue(targetQueue.getRunnableAppSchedulables().contains(app)); assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage()); assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage()); assertEquals(0, oldQueue.getNumRunnableApps()); @@ -3224,17 +3219,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); - FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); - assertTrue(oldQueue.getNonRunnableAppSchedulables() - .contains(app.getAppSchedulable())); + FSAppAttempt app = scheduler.getSchedulerApp(appAttId); + assertTrue(oldQueue.getNonRunnableAppSchedulables().contains(app)); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); - assertFalse(oldQueue.getNonRunnableAppSchedulables() - .contains(app.getAppSchedulable())); - assertFalse(targetQueue.getNonRunnableAppSchedulables() - .contains(app.getAppSchedulable())); - assertTrue(targetQueue.getRunnableAppSchedulables() - .contains(app.getAppSchedulable())); + assertFalse(oldQueue.getNonRunnableAppSchedulables().contains(app)); + assertFalse(targetQueue.getNonRunnableAppSchedulables().contains(app)); + assertTrue(targetQueue.getRunnableAppSchedulables().contains(app)); assertEquals(1, targetQueue.getNumRunnableApps()); assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index cc738f5eb01..20ff2c9cd25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -42,12 +42,13 @@ public class TestMaxRunningAppsEnforcer { private int appNum; private TestFairScheduler.MockClock clock; private RMContext rmContext; + private FairScheduler scheduler; @Before public void setup() throws Exception { Configuration conf = new Configuration(); clock = new TestFairScheduler.MockClock(); - FairScheduler scheduler = mock(FairScheduler.class); + scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); @@ -65,11 +66,11 @@ public class TestMaxRunningAppsEnforcer { when(rmContext.getEpoch()).thenReturn(0); } - private FSSchedulerApp addApp(FSLeafQueue queue, String user) { + private FSAppAttempt addApp(FSLeafQueue queue, String user) { ApplicationId appId = ApplicationId.newInstance(0l, appNum++); ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 0); boolean runnable = maxAppsEnforcer.canAppBeRunnable(queue, user); - FSSchedulerApp app = new FSSchedulerApp(attId, user, queue, null, + FSAppAttempt app = new FSAppAttempt(scheduler, attId, user, queue, null, rmContext); queue.addApp(app, runnable); if (runnable) { @@ -80,7 +81,7 @@ public class TestMaxRunningAppsEnforcer { return app; } - private void removeApp(FSSchedulerApp app) { + private void removeApp(FSAppAttempt app) { app.getQueue().removeApp(app); maxAppsEnforcer.untrackRunnableApp(app); maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue()); @@ -93,7 +94,7 @@ public class TestMaxRunningAppsEnforcer { queueMaxApps.put("root", 2); queueMaxApps.put("root.queue1", 1); queueMaxApps.put("root.queue2", 1); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); assertEquals(1, leaf1.getRunnableAppSchedulables().size()); @@ -110,7 +111,7 @@ public class TestMaxRunningAppsEnforcer { FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); assertEquals(1, leaf1.getRunnableAppSchedulables().size()); @@ -128,7 +129,7 @@ public class TestMaxRunningAppsEnforcer { FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true); queueMaxApps.put("root.queue1.leaf1", 2); userMaxApps.put("user1", 1); - FSSchedulerApp app1 = addApp(leaf1, "user1"); + FSAppAttempt app1 = addApp(leaf1, "user1"); addApp(leaf1, "user2"); addApp(leaf1, "user3"); addApp(leaf2, "user1"); @@ -147,7 +148,7 @@ public class TestMaxRunningAppsEnforcer { FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); clock.tick(20); @@ -167,7 +168,7 @@ public class TestMaxRunningAppsEnforcer { FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); @@ -182,21 +183,18 @@ public class TestMaxRunningAppsEnforcer { @Test public void testMultiListStartTimeIteratorEmptyAppLists() { - List> lists = new ArrayList>(); - lists.add(Arrays.asList(mockAppSched(1))); - lists.add(Arrays.asList(mockAppSched(2))); - Iterator iter = + List> lists = new ArrayList>(); + lists.add(Arrays.asList(mockAppAttempt(1))); + lists.add(Arrays.asList(mockAppAttempt(2))); + Iterator iter = new MaxRunningAppsEnforcer.MultiListStartTimeIterator(lists); - assertEquals(1, iter.next().getAppSchedulable().getStartTime()); - assertEquals(2, iter.next().getAppSchedulable().getStartTime()); + assertEquals(1, iter.next().getStartTime()); + assertEquals(2, iter.next().getStartTime()); } - private AppSchedulable mockAppSched(long startTime) { - AppSchedulable appSched = mock(AppSchedulable.class); - when(appSched.getStartTime()).thenReturn(startTime); - FSSchedulerApp schedApp = mock(FSSchedulerApp.class); - when(schedApp.getAppSchedulable()).thenReturn(appSched); - when(appSched.getApp()).thenReturn(schedApp); - return appSched; + private FSAppAttempt mockAppAttempt(long startTime) { + FSAppAttempt schedApp = mock(FSAppAttempt.class); + when(schedApp.getStartTime()).thenReturn(startTime); + return schedApp; } } From a72fba5853d0537bcdbd7851181129173e440dbb Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 12 Aug 2014 21:49:48 +0000 Subject: [PATCH 06/67] HDFS-6836. HDFS INFO logging is verbose & uses file appenders. (Contributed by Xiaoyu Yao) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617603 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/datanode/BlockSender.java | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e448ca6727d..8148052005b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -393,6 +393,9 @@ Release 2.6.0 - UNRELEASED HDFS-6838. Code cleanup for unnecessary INode replacement. (Jing Zhao via wheat9) + HDFS-6836. HDFS INFO logging is verbose & uses file appenders. (Xiaoyu + Yao via Arpit Agarwal) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index 86d88c2a44e..febf2de0c3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -687,7 +687,7 @@ class BlockSender implements java.io.Closeable { // Trigger readahead of beginning of file if configured. manageOsCache(); - final long startTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime() : 0; + final long startTime = ClientTraceLog.isDebugEnabled() ? System.nanoTime() : 0; try { int maxChunksPerPacket; int pktBufSize = PacketHeader.PKT_MAX_HEADER_LEN; @@ -733,9 +733,9 @@ class BlockSender implements java.io.Closeable { sentEntireByteRange = true; } } finally { - if (clientTraceFmt != null) { + if ((clientTraceFmt != null) && ClientTraceLog.isDebugEnabled()) { final long endTime = System.nanoTime(); - ClientTraceLog.info(String.format(clientTraceFmt, totalRead, + ClientTraceLog.debug(String.format(clientTraceFmt, totalRead, initialOffset, endTime - startTime)); } close(); From d687f6f68990bfe5fb819e36a91a25aef2c3a63f Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 12 Aug 2014 22:30:48 +0000 Subject: [PATCH 07/67] HADOOP-10851. NetgroupCache does not remove group memberships. (Contributed by Benoy Antony) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617612 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/security/NetgroupCache.java | 17 +-- .../hadoop/security/TestNetgroupCache.java | 127 ++++++++++++++++++ 3 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestNetgroupCache.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 08c981e1e58..c72b3de3d82 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -563,6 +563,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10402. Configuration.getValByRegex does not substitute for variables. (Robert Kanter via kasha) + HADOOP-10851. NetgroupCache does not remove group memberships. (Benoy + Antony via Arpit Agarwal) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java index d07ae2bd018..bd9c448da7f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java @@ -27,12 +27,9 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - /** * Class that caches the netgroups and inverts group-to-user map - * to user-to-group map, primarily intented for use with + * to user-to-group map, primarily intended for use with * netgroups (as returned by getent netgrgoup) which only returns * group to user mapping. */ @@ -69,9 +66,7 @@ public class NetgroupCache { } } if(userToNetgroupsMap.containsKey(user)) { - for(String netgroup : userToNetgroupsMap.get(user)) { - groups.add(netgroup); - } + groups.addAll(userToNetgroupsMap.get(user)); } } @@ -99,6 +94,7 @@ public class NetgroupCache { */ public static void clear() { netgroupToUsersMap.clear(); + userToNetgroupsMap.clear(); } /** @@ -108,12 +104,7 @@ public class NetgroupCache { * @param users list of users for a given group */ public static void add(String group, List users) { - if(!isCached(group)) { - netgroupToUsersMap.put(group, new HashSet()); - for(String user: users) { - netgroupToUsersMap.get(group).add(user); - } - } + netgroupToUsersMap.put(group, new HashSet(users)); netgroupToUsersMapUpdated = true; // at the end to avoid race } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestNetgroupCache.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestNetgroupCache.java new file mode 100644 index 00000000000..bd95422e651 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestNetgroupCache.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.security; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.After; +import org.junit.Test; + +public class TestNetgroupCache { + + private static final String USER1 = "user1"; + private static final String USER2 = "user2"; + private static final String USER3 = "user3"; + private static final String GROUP1 = "group1"; + private static final String GROUP2 = "group2"; + + @After + public void teardown() { + NetgroupCache.clear(); + } + + /** + * Cache two groups with a set of users. + * Test membership correctness. + */ + @Test + public void testMembership() { + List users = new ArrayList(); + users.add(USER1); + users.add(USER2); + NetgroupCache.add(GROUP1, users); + users = new ArrayList(); + users.add(USER1); + users.add(USER3); + NetgroupCache.add(GROUP2, users); + verifyGroupMembership(USER1, 2, GROUP1); + verifyGroupMembership(USER1, 2, GROUP2); + verifyGroupMembership(USER2, 1, GROUP1); + verifyGroupMembership(USER3, 1, GROUP2); + } + + /** + * Cache a group with a set of users. + * Test membership correctness. + * Clear cache, remove a user from the group and cache the group + * Test membership correctness. + */ + @Test + public void testUserRemoval() { + List users = new ArrayList(); + users.add(USER1); + users.add(USER2); + NetgroupCache.add(GROUP1, users); + verifyGroupMembership(USER1, 1, GROUP1); + verifyGroupMembership(USER2, 1, GROUP1); + users.remove(USER2); + NetgroupCache.clear(); + NetgroupCache.add(GROUP1, users); + verifyGroupMembership(USER1, 1, GROUP1); + verifyGroupMembership(USER2, 0, null); + } + + /** + * Cache two groups with a set of users. + * Test membership correctness. + * Clear cache, cache only one group. + * Test membership correctness. + */ + @Test + public void testGroupRemoval() { + List users = new ArrayList(); + users.add(USER1); + users.add(USER2); + NetgroupCache.add(GROUP1, users); + users = new ArrayList(); + users.add(USER1); + users.add(USER3); + NetgroupCache.add(GROUP2, users); + verifyGroupMembership(USER1, 2, GROUP1); + verifyGroupMembership(USER1, 2, GROUP2); + verifyGroupMembership(USER2, 1, GROUP1); + verifyGroupMembership(USER3, 1, GROUP2); + NetgroupCache.clear(); + users = new ArrayList(); + users.add(USER1); + users.add(USER2); + NetgroupCache.add(GROUP1, users); + verifyGroupMembership(USER1, 1, GROUP1); + verifyGroupMembership(USER2, 1, GROUP1); + verifyGroupMembership(USER3, 0, null); + } + + private void verifyGroupMembership(String user, int size, String group) { + List groups = new ArrayList(); + NetgroupCache.getNetgroups(user, groups); + assertEquals(size, groups.size()); + if (size > 0) { + boolean present = false; + for (String groupEntry:groups) { + if (groupEntry.equals(group)) { + present = true; + break; + } + } + assertTrue(present); + } + } +} From 4239695588f0e133bd514955c3404d3301eac6ce Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Tue, 12 Aug 2014 22:51:57 +0000 Subject: [PATCH 08/67] YARN-2399. Delete old versions of files. FairScheduler: Merge AppSchedulable and FSSchedulerApp into FSAppAttempt. (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617619 13f79535-47bb-0310-9956-ffa450edef68 --- .../scheduler/fair/AppSchedulable.java | 463 ------------------ .../scheduler/fair/FSSchedulerApp.java | 360 -------------- .../scheduler/fair/TestFSSchedulerApp.java | 191 -------- 3 files changed, 1014 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java deleted file mode 100644 index 0c36c55892b..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ /dev/null @@ -1,463 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.apache.hadoop.yarn.util.resource.Resources; - -@Private -@Unstable -public class AppSchedulable extends Schedulable { - private static final DefaultResourceCalculator RESOURCE_CALCULATOR - = new DefaultResourceCalculator(); - - private FairScheduler scheduler; - private FSSchedulerApp app; - private Resource demand = Resources.createResource(0); - private long startTime; - private static final Log LOG = LogFactory.getLog(AppSchedulable.class); - private FSLeafQueue queue; - private RMContainerTokenSecretManager containerTokenSecretManager; - private Priority priority; - private ResourceWeights resourceWeights; - - private RMContainerComparator comparator = new RMContainerComparator(); - - public AppSchedulable(FairScheduler scheduler, FSSchedulerApp app, FSLeafQueue queue) { - this.scheduler = scheduler; - this.app = app; - this.startTime = scheduler.getClock().getTime(); - this.queue = queue; - this.containerTokenSecretManager = scheduler. - getContainerTokenSecretManager(); - this.priority = Priority.newInstance(1); - this.resourceWeights = new ResourceWeights(); - } - - @Override - public String getName() { - return app.getApplicationId().toString(); - } - - public FSSchedulerApp getApp() { - return app; - } - - public ResourceWeights getResourceWeights() { - return resourceWeights; - } - - @Override - public void updateDemand() { - demand = Resources.createResource(0); - // Demand is current consumption plus outstanding requests - Resources.addTo(demand, app.getCurrentConsumption()); - - // Add up outstanding resource requests - synchronized (app) { - for (Priority p : app.getPriorities()) { - for (ResourceRequest r : app.getResourceRequests(p).values()) { - Resource total = Resources.multiply(r.getCapability(), r.getNumContainers()); - Resources.addTo(demand, total); - } - } - } - } - - @Override - public Resource getDemand() { - return demand; - } - - @Override - public long getStartTime() { - return startTime; - } - - @Override - public Resource getResourceUsage() { - // Here the getPreemptedResources() always return zero, except in - // a preemption round - return Resources.subtract(app.getCurrentConsumption(), - app.getPreemptedResources()); - } - - - @Override - public Resource getMinShare() { - return Resources.none(); - } - - @Override - public Resource getMaxShare() { - return Resources.unbounded(); - } - - /** - * Get metrics reference from containing queue. - */ - public QueueMetrics getMetrics() { - return queue.getMetrics(); - } - - @Override - public ResourceWeights getWeights() { - return scheduler.getAppWeight(this); - } - - @Override - public Priority getPriority() { - // Right now per-app priorities are not passed to scheduler, - // so everyone has the same priority. - return priority; - } - - /** - * Create and return a container object reflecting an allocation for the - * given appliction on the given node with the given capability and - * priority. - */ - public Container createContainer( - FSSchedulerApp application, FSSchedulerNode node, - Resource capability, Priority priority) { - - NodeId nodeId = node.getRMNode().getNodeID(); - ContainerId containerId = BuilderUtils.newContainerId(application - .getApplicationAttemptId(), application.getNewContainerId()); - - // Create the container - Container container = - BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() - .getHttpAddress(), capability, priority, null); - - return container; - } - - /** - * Reserve a spot for {@code container} on this {@code node}. If - * the container is {@code alreadyReserved} on the node, simply - * update relevant bookeeping. This dispatches ro relevant handlers - * in the {@link FSSchedulerNode} and {@link SchedulerApp} classes. - */ - private void reserve(Priority priority, FSSchedulerNode node, - Container container, boolean alreadyReserved) { - LOG.info("Making reservation: node=" + node.getNodeName() + - " app_id=" + app.getApplicationId()); - if (!alreadyReserved) { - getMetrics().reserveResource(app.getUser(), container.getResource()); - RMContainer rmContainer = app.reserve(node, priority, null, - container); - node.reserveResource(app, priority, rmContainer); - } - - else { - RMContainer rmContainer = node.getReservedContainer(); - app.reserve(node, priority, rmContainer, container); - node.reserveResource(app, priority, rmContainer); - } - } - - /** - * Remove the reservation on {@code node} at the given - * {@link Priority}. This dispatches to the SchedulerApp and SchedulerNode - * handlers for an unreservation. - */ - public void unreserve(Priority priority, FSSchedulerNode node) { - RMContainer rmContainer = node.getReservedContainer(); - app.unreserve(node, priority); - node.unreserveResource(app); - getMetrics().unreserveResource( - app.getUser(), rmContainer.getContainer().getResource()); - } - - /** - * Assign a container to this node to facilitate {@code request}. If node does - * not have enough memory, create a reservation. This is called once we are - * sure the particular request should be facilitated by this node. - * - * @param node - * The node to try placing the container on. - * @param priority - * The requested priority for the container. - * @param request - * The ResourceRequest we're trying to satisfy. - * @param type - * The locality of the assignment. - * @param reserved - * Whether there's already a container reserved for this app on the node. - * @return - * If an assignment was made, returns the resources allocated to the - * container. If a reservation was made, returns - * FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was - * made, returns an empty resource. - */ - private Resource assignContainer(FSSchedulerNode node, - ResourceRequest request, NodeType type, - boolean reserved) { - - // How much does this request need? - Resource capability = request.getCapability(); - - // How much does the node have? - Resource available = node.getAvailableResource(); - - Container container = null; - if (reserved) { - container = node.getReservedContainer().getContainer(); - } else { - container = createContainer(app, node, capability, request.getPriority()); - } - - // Can we allocate a container on this node? - if (Resources.fitsIn(capability, available)) { - // Inform the application of the new container for this request - RMContainer allocatedContainer = - app.allocate(type, node, request.getPriority(), request, container); - if (allocatedContainer == null) { - // Did the application need this resource? - if (reserved) { - unreserve(request.getPriority(), node); - } - return Resources.none(); - } - - // If we had previously made a reservation, delete it - if (reserved) { - unreserve(request.getPriority(), node); - } - - // Inform the node - node.allocateContainer(allocatedContainer); - - // If this container is used to run AM, update the leaf queue's AM usage - if (app.getLiveContainers().size() == 1 && - !app.getUnmanagedAM()) { - queue.addAMResourceUsage(container.getResource()); - app.setAmRunning(true); - } - - return container.getResource(); - } else { - // The desired container won't fit here, so reserve - reserve(request.getPriority(), node, container, reserved); - - return FairScheduler.CONTAINER_RESERVED; - } - } - - private Resource assignContainer(FSSchedulerNode node, boolean reserved) { - if (LOG.isDebugEnabled()) { - LOG.debug("Node offered to app: " + getName() + " reserved: " + reserved); - } - - Collection prioritiesToTry = (reserved) ? - Arrays.asList(node.getReservedContainer().getReservedPriority()) : - app.getPriorities(); - - // For each priority, see if we can schedule a node local, rack local - // or off-switch request. Rack of off-switch requests may be delayed - // (not scheduled) in order to promote better locality. - synchronized (app) { - for (Priority priority : prioritiesToTry) { - if (app.getTotalRequiredResources(priority) <= 0 || - !hasContainerForNode(priority, node)) { - continue; - } - - app.addSchedulingOpportunity(priority); - - // Check the AM resource usage for the leaf queue - if (app.getLiveContainers().size() == 0 - && !app.getUnmanagedAM()) { - if (!queue.canRunAppAM(app.getAMResource())) { - return Resources.none(); - } - } - - ResourceRequest rackLocalRequest = app.getResourceRequest(priority, - node.getRackName()); - ResourceRequest localRequest = app.getResourceRequest(priority, - node.getNodeName()); - - if (localRequest != null && !localRequest.getRelaxLocality()) { - LOG.warn("Relax locality off is not supported on local request: " - + localRequest); - } - - NodeType allowedLocality; - if (scheduler.isContinuousSchedulingEnabled()) { - allowedLocality = app.getAllowedLocalityLevelByTime(priority, - scheduler.getNodeLocalityDelayMs(), - scheduler.getRackLocalityDelayMs(), - scheduler.getClock().getTime()); - } else { - allowedLocality = app.getAllowedLocalityLevel(priority, - scheduler.getNumClusterNodes(), - scheduler.getNodeLocalityThreshold(), - scheduler.getRackLocalityThreshold()); - } - - if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 - && localRequest != null && localRequest.getNumContainers() != 0) { - return assignContainer(node, localRequest, - NodeType.NODE_LOCAL, reserved); - } - - if (rackLocalRequest != null && !rackLocalRequest.getRelaxLocality()) { - continue; - } - - if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 - && (allowedLocality.equals(NodeType.RACK_LOCAL) || - allowedLocality.equals(NodeType.OFF_SWITCH))) { - return assignContainer(node, rackLocalRequest, - NodeType.RACK_LOCAL, reserved); - } - - ResourceRequest offSwitchRequest = app.getResourceRequest(priority, - ResourceRequest.ANY); - if (offSwitchRequest != null && !offSwitchRequest.getRelaxLocality()) { - continue; - } - - if (offSwitchRequest != null && offSwitchRequest.getNumContainers() != 0 - && allowedLocality.equals(NodeType.OFF_SWITCH)) { - return assignContainer(node, offSwitchRequest, - NodeType.OFF_SWITCH, reserved); - } - } - } - return Resources.none(); - } - - /** - * Called when this application already has an existing reservation on the - * given node. Sees whether we can turn the reservation into an allocation. - * Also checks whether the application needs the reservation anymore, and - * releases it if not. - * - * @param node - * Node that the application has an existing reservation on - */ - public Resource assignReservedContainer(FSSchedulerNode node) { - RMContainer rmContainer = node.getReservedContainer(); - Priority priority = rmContainer.getReservedPriority(); - - // Make sure the application still needs requests at this priority - if (app.getTotalRequiredResources(priority) == 0) { - unreserve(priority, node); - return Resources.none(); - } - - // Fail early if the reserved container won't fit. - // Note that we have an assumption here that there's only one container size - // per priority. - if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(), - node.getAvailableResource())) { - return Resources.none(); - } - - return assignContainer(node, true); - } - - @Override - public Resource assignContainer(FSSchedulerNode node) { - return assignContainer(node, false); - } - - /** - * Preempt a running container according to the priority - */ - @Override - public RMContainer preemptContainer() { - if (LOG.isDebugEnabled()) { - LOG.debug("App " + getName() + " is going to preempt a running " + - "container"); - } - - RMContainer toBePreempted = null; - for (RMContainer container : app.getLiveContainers()) { - if (! app.getPreemptionContainers().contains(container) && - (toBePreempted == null || - comparator.compare(toBePreempted, container) > 0)) { - toBePreempted = container; - } - } - return toBePreempted; - } - - /** - * Whether this app has containers requests that could be satisfied on the - * given node, if the node had full space. - */ - public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) { - ResourceRequest anyRequest = app.getResourceRequest(prio, ResourceRequest.ANY); - ResourceRequest rackRequest = app.getResourceRequest(prio, node.getRackName()); - ResourceRequest nodeRequest = app.getResourceRequest(prio, node.getNodeName()); - - return - // There must be outstanding requests at the given priority: - anyRequest != null && anyRequest.getNumContainers() > 0 && - // If locality relaxation is turned off at *-level, there must be a - // non-zero request for the node's rack: - (anyRequest.getRelaxLocality() || - (rackRequest != null && rackRequest.getNumContainers() > 0)) && - // If locality relaxation is turned off at rack-level, there must be a - // non-zero request at the node: - (rackRequest == null || rackRequest.getRelaxLocality() || - (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && - // The requested container must be able to fit on the node: - Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, - anyRequest.getCapability(), node.getRMNode().getTotalCapability()); - } - - static class RMContainerComparator implements Comparator, - Serializable { - @Override - public int compare(RMContainer c1, RMContainer c2) { - int ret = c1.getContainer().getPriority().compareTo( - c2.getContainer().getPriority()); - if (ret == 0) { - return c2.getContainerId().compareTo(c1.getContainerId()); - } - return ret; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java deleted file mode 100644 index 20cf3952d2d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ /dev/null @@ -1,360 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; -import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.util.resource.Resources; - -/** - * Represents an application attempt from the viewpoint of the Fair Scheduler. - */ -@Private -@Unstable -public class FSSchedulerApp extends SchedulerApplicationAttempt { - - private static final Log LOG = LogFactory.getLog(FSSchedulerApp.class); - - private AppSchedulable appSchedulable; - - final Map preemptionMap = new HashMap(); - - private Resource preemptedResources = Resources.createResource(0); - - public FSSchedulerApp(ApplicationAttemptId applicationAttemptId, - String user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, - RMContext rmContext) { - super(applicationAttemptId, user, queue, activeUsersManager, rmContext); - } - - public void setAppSchedulable(AppSchedulable appSchedulable) { - this.appSchedulable = appSchedulable; - } - - public AppSchedulable getAppSchedulable() { - return appSchedulable; - } - - synchronized public void containerCompleted(RMContainer rmContainer, - ContainerStatus containerStatus, RMContainerEventType event) { - - Container container = rmContainer.getContainer(); - ContainerId containerId = container.getId(); - - // Remove from the list of newly allocated containers if found - newlyAllocatedContainers.remove(rmContainer); - - // Inform the container - rmContainer.handle( - new RMContainerFinishedEvent( - containerId, - containerStatus, - event) - ); - LOG.info("Completed container: " + rmContainer.getContainerId() + - " in state: " + rmContainer.getState() + " event:" + event); - - // Remove from the list of containers - liveContainers.remove(rmContainer.getContainerId()); - - RMAuditLogger.logSuccess(getUser(), - AuditConstants.RELEASE_CONTAINER, "SchedulerApp", - getApplicationId(), containerId); - - // Update usage metrics - Resource containerResource = rmContainer.getContainer().getResource(); - queue.getMetrics().releaseResources(getUser(), 1, containerResource); - Resources.subtractFrom(currentConsumption, containerResource); - - // remove from preemption map if it is completed - preemptionMap.remove(rmContainer); - } - - public synchronized void unreserve(FSSchedulerNode node, Priority priority) { - Map reservedContainers = - this.reservedContainers.get(priority); - RMContainer reservedContainer = reservedContainers.remove(node.getNodeID()); - if (reservedContainers.isEmpty()) { - this.reservedContainers.remove(priority); - } - - // Reset the re-reservation count - resetReReservations(priority); - - Resource resource = reservedContainer.getContainer().getResource(); - Resources.subtractFrom(currentReservation, resource); - - LOG.info("Application " + getApplicationId() + " unreserved " + " on node " - + node + ", currently has " + reservedContainers.size() + " at priority " - + priority + "; currentReservation " + currentReservation); - } - - public synchronized float getLocalityWaitFactor( - Priority priority, int clusterNodes) { - // Estimate: Required unique resources (i.e. hosts + racks) - int requiredResources = - Math.max(this.getResourceRequests(priority).size() - 1, 0); - - // waitFactor can't be more than '1' - // i.e. no point skipping more than clustersize opportunities - return Math.min(((float)requiredResources / clusterNodes), 1.0f); - } - - /** - * Delay scheduling: We often want to prioritize scheduling of node-local - * containers over rack-local or off-switch containers. To acheive this - * we first only allow node-local assigments for a given prioirty level, - * then relax the locality threshold once we've had a long enough period - * without succesfully scheduling. We measure both the number of "missed" - * scheduling opportunities since the last container was scheduled - * at the current allowed level and the time since the last container - * was scheduled. Currently we use only the former. - */ - - // Current locality threshold - final Map allowedLocalityLevel = new HashMap< - Priority, NodeType>(); - - /** - * Return the level at which we are allowed to schedule containers, given the - * current size of the cluster and thresholds indicating how many nodes to - * fail at (as a fraction of cluster size) before relaxing scheduling - * constraints. - */ - public synchronized NodeType getAllowedLocalityLevel(Priority priority, - int numNodes, double nodeLocalityThreshold, double rackLocalityThreshold) { - // upper limit on threshold - if (nodeLocalityThreshold > 1.0) { nodeLocalityThreshold = 1.0; } - if (rackLocalityThreshold > 1.0) { rackLocalityThreshold = 1.0; } - - // If delay scheduling is not being used, can schedule anywhere - if (nodeLocalityThreshold < 0.0 || rackLocalityThreshold < 0.0) { - return NodeType.OFF_SWITCH; - } - - // Default level is NODE_LOCAL - if (!allowedLocalityLevel.containsKey(priority)) { - allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL); - return NodeType.NODE_LOCAL; - } - - NodeType allowed = allowedLocalityLevel.get(priority); - - // If level is already most liberal, we're done - if (allowed.equals(NodeType.OFF_SWITCH)) return NodeType.OFF_SWITCH; - - double threshold = allowed.equals(NodeType.NODE_LOCAL) ? nodeLocalityThreshold : - rackLocalityThreshold; - - // Relax locality constraints once we've surpassed threshold. - if (getSchedulingOpportunities(priority) > (numNodes * threshold)) { - if (allowed.equals(NodeType.NODE_LOCAL)) { - allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL); - resetSchedulingOpportunities(priority); - } - else if (allowed.equals(NodeType.RACK_LOCAL)) { - allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH); - resetSchedulingOpportunities(priority); - } - } - return allowedLocalityLevel.get(priority); - } - - /** - * Return the level at which we are allowed to schedule containers. - * Given the thresholds indicating how much time passed before relaxing - * scheduling constraints. - */ - public synchronized NodeType getAllowedLocalityLevelByTime(Priority priority, - long nodeLocalityDelayMs, long rackLocalityDelayMs, - long currentTimeMs) { - - // if not being used, can schedule anywhere - if (nodeLocalityDelayMs < 0 || rackLocalityDelayMs < 0) { - return NodeType.OFF_SWITCH; - } - - // default level is NODE_LOCAL - if (! allowedLocalityLevel.containsKey(priority)) { - allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL); - return NodeType.NODE_LOCAL; - } - - NodeType allowed = allowedLocalityLevel.get(priority); - - // if level is already most liberal, we're done - if (allowed.equals(NodeType.OFF_SWITCH)) { - return NodeType.OFF_SWITCH; - } - - // check waiting time - long waitTime = currentTimeMs; - if (lastScheduledContainer.containsKey(priority)) { - waitTime -= lastScheduledContainer.get(priority); - } else { - waitTime -= appSchedulable.getStartTime(); - } - - long thresholdTime = allowed.equals(NodeType.NODE_LOCAL) ? - nodeLocalityDelayMs : rackLocalityDelayMs; - - if (waitTime > thresholdTime) { - if (allowed.equals(NodeType.NODE_LOCAL)) { - allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL); - resetSchedulingOpportunities(priority, currentTimeMs); - } else if (allowed.equals(NodeType.RACK_LOCAL)) { - allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH); - resetSchedulingOpportunities(priority, currentTimeMs); - } - } - return allowedLocalityLevel.get(priority); - } - - synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node, - Priority priority, ResourceRequest request, - Container container) { - // Update allowed locality level - NodeType allowed = allowedLocalityLevel.get(priority); - if (allowed != null) { - if (allowed.equals(NodeType.OFF_SWITCH) && - (type.equals(NodeType.NODE_LOCAL) || - type.equals(NodeType.RACK_LOCAL))) { - this.resetAllowedLocalityLevel(priority, type); - } - else if (allowed.equals(NodeType.RACK_LOCAL) && - type.equals(NodeType.NODE_LOCAL)) { - this.resetAllowedLocalityLevel(priority, type); - } - } - - // Required sanity check - AM can call 'allocate' to update resource - // request without locking the scheduler, hence we need to check - if (getTotalRequiredResources(priority) <= 0) { - return null; - } - - // Create RMContainer - RMContainer rmContainer = new RMContainerImpl(container, - getApplicationAttemptId(), node.getNodeID(), - appSchedulingInfo.getUser(), rmContext); - - // Add it to allContainers list. - newlyAllocatedContainers.add(rmContainer); - liveContainers.put(container.getId(), rmContainer); - - // Update consumption and track allocations - List resourceRequestList = appSchedulingInfo.allocate( - type, node, priority, request, container); - Resources.addTo(currentConsumption, container.getResource()); - - // Update resource requests related to "request" and store in RMContainer - ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList); - - // Inform the container - rmContainer.handle( - new RMContainerEvent(container.getId(), RMContainerEventType.START)); - - if (LOG.isDebugEnabled()) { - LOG.debug("allocate: applicationAttemptId=" - + container.getId().getApplicationAttemptId() - + " container=" + container.getId() + " host=" - + container.getNodeId().getHost() + " type=" + type); - } - RMAuditLogger.logSuccess(getUser(), - AuditConstants.ALLOC_CONTAINER, "SchedulerApp", - getApplicationId(), container.getId()); - - return rmContainer; - } - - /** - * Should be called when the scheduler assigns a container at a higher - * degree of locality than the current threshold. Reset the allowed locality - * level to a higher degree of locality. - */ - public synchronized void resetAllowedLocalityLevel(Priority priority, - NodeType level) { - NodeType old = allowedLocalityLevel.get(priority); - LOG.info("Raising locality level from " + old + " to " + level + " at " + - " priority " + priority); - allowedLocalityLevel.put(priority, level); - } - - // related methods - public void addPreemption(RMContainer container, long time) { - assert preemptionMap.get(container) == null; - preemptionMap.put(container, time); - Resources.addTo(preemptedResources, container.getAllocatedResource()); - } - - public Long getContainerPreemptionTime(RMContainer container) { - return preemptionMap.get(container); - } - - public Set getPreemptionContainers() { - return preemptionMap.keySet(); - } - - @Override - public FSLeafQueue getQueue() { - return (FSLeafQueue)super.getQueue(); - } - - public Resource getPreemptedResources() { - return preemptedResources; - } - - public void resetPreemptedResources() { - preemptedResources = Resources.createResource(0); - for (RMContainer container : getPreemptionContainers()) { - Resources.addTo(preemptedResources, container.getAllocatedResource()); - } - } - - public void clearPreemptedResources() { - preemptedResources.setMemory(0); - preemptedResources.setVirtualCores(0); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java deleted file mode 100644 index 2d5a6d4bc8c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import static org.junit.Assert.assertEquals; - -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; -import org.apache.hadoop.yarn.util.Clock; -import org.junit.Test; -import org.mockito.Mockito; - -public class TestFSSchedulerApp { - - private class MockClock implements Clock { - private long time = 0; - @Override - public long getTime() { - return time; - } - - public void tick(int seconds) { - time = time + seconds * 1000; - } - - } - - private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { - ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); - ApplicationAttemptId attId = - ApplicationAttemptId.newInstance(appIdImpl, attemptId); - return attId; - } - - @Test - public void testDelayScheduling() { - FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); - Priority prio = Mockito.mock(Priority.class); - Mockito.when(prio.getPriority()).thenReturn(1); - double nodeLocalityThreshold = .5; - double rackLocalityThreshold = .6; - - ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); - RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getEpoch()).thenReturn(0); - FSSchedulerApp schedulerApp = - new FSSchedulerApp(applicationAttemptId, "user1", queue , null, - rmContext); - - // Default level should be node-local - assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - - // First five scheduling opportunities should remain node local - for (int i = 0; i < 5; i++) { - schedulerApp.addSchedulingOpportunity(prio); - assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - } - - // After five it should switch to rack local - schedulerApp.addSchedulingOpportunity(prio); - assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - - // Manually set back to node local - schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL); - schedulerApp.resetSchedulingOpportunities(prio); - assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - - // Now escalate again to rack-local, then to off-switch - for (int i = 0; i < 5; i++) { - schedulerApp.addSchedulingOpportunity(prio); - assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - } - - schedulerApp.addSchedulingOpportunity(prio); - assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - - for (int i = 0; i < 6; i++) { - schedulerApp.addSchedulingOpportunity(prio); - assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - } - - schedulerApp.addSchedulingOpportunity(prio); - assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel( - prio, 10, nodeLocalityThreshold, rackLocalityThreshold)); - } - - @Test - public void testDelaySchedulingForContinuousScheduling() - throws InterruptedException { - FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); - Priority prio = Mockito.mock(Priority.class); - Mockito.when(prio.getPriority()).thenReturn(1); - - MockClock clock = new MockClock(); - - long nodeLocalityDelayMs = 5 * 1000L; // 5 seconds - long rackLocalityDelayMs = 6 * 1000L; // 6 seconds - - RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getEpoch()).thenReturn(0); - ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); - FSSchedulerApp schedulerApp = - new FSSchedulerApp(applicationAttemptId, "user1", queue, - null, rmContext); - AppSchedulable appSchedulable = Mockito.mock(AppSchedulable.class); - long startTime = clock.getTime(); - Mockito.when(appSchedulable.getStartTime()).thenReturn(startTime); - schedulerApp.setAppSchedulable(appSchedulable); - - // Default level should be node-local - assertEquals(NodeType.NODE_LOCAL, - schedulerApp.getAllowedLocalityLevelByTime(prio, - nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); - - // after 4 seconds should remain node local - clock.tick(4); - assertEquals(NodeType.NODE_LOCAL, - schedulerApp.getAllowedLocalityLevelByTime(prio, - nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); - - // after 6 seconds should switch to rack local - clock.tick(2); - assertEquals(NodeType.RACK_LOCAL, - schedulerApp.getAllowedLocalityLevelByTime(prio, - nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); - - // manually set back to node local - schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL); - schedulerApp.resetSchedulingOpportunities(prio, clock.getTime()); - assertEquals(NodeType.NODE_LOCAL, - schedulerApp.getAllowedLocalityLevelByTime(prio, - nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); - - // Now escalate again to rack-local, then to off-switch - clock.tick(6); - assertEquals(NodeType.RACK_LOCAL, - schedulerApp.getAllowedLocalityLevelByTime(prio, - nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); - - clock.tick(7); - assertEquals(NodeType.OFF_SWITCH, - schedulerApp.getAllowedLocalityLevelByTime(prio, - nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime())); - } - - @Test - /** - * Ensure that when negative paramaters are given (signaling delay scheduling - * no tin use), the least restrictive locality level is returned. - */ - public void testLocalityLevelWithoutDelays() { - FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); - Priority prio = Mockito.mock(Priority.class); - Mockito.when(prio.getPriority()).thenReturn(1); - - RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getEpoch()).thenReturn(0); - ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); - FSSchedulerApp schedulerApp = - new FSSchedulerApp(applicationAttemptId, "user1", queue , null, - rmContext); - assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel( - prio, 10, -1.0, -1.0)); - } -} From f7ac928a7c4b44d15d00ddace27af3738fdd01a1 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Tue, 12 Aug 2014 23:03:06 +0000 Subject: [PATCH 09/67] HADOOP-10962. Flags for posix_fadvise are not valid in some architectures (David Villegas via Colin Patrick McCabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617621 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/io/nativeio/NativeIO.c | 31 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c72b3de3d82..38bef36de79 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -566,6 +566,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10851. NetgroupCache does not remove group memberships. (Benoy Antony via Arpit Agarwal) + HADOOP-10962. Flags for posix_fadvise are not valid in some architectures + (David Villegas via Colin Patrick McCabe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index 23513bfd667..fa95047394f 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -171,6 +171,35 @@ static void nioe_deinit(JNIEnv *env) { nioe_ctor = NULL; } +/* + * Compatibility mapping for fadvise flags. Return the proper value from fnctl.h. + * If the value is not known, return the argument unchanged. + */ +static int map_fadvise_flag(jint flag) { + switch(flag) { + case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_NORMAL: + return POSIX_FADV_NORMAL; + break; + case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_RANDOM: + return POSIX_FADV_RANDOM; + break; + case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_SEQUENTIAL: + return POSIX_FADV_SEQUENTIAL; + break; + case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_WILLNEED: + return POSIX_FADV_WILLNEED; + break; + case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_DONTNEED: + return POSIX_FADV_DONTNEED; + break; + case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_NOREUSE: + return POSIX_FADV_NOREUSE; + break; + default: + return flag; + } +} + /* * private static native void initNative(); * @@ -303,7 +332,7 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_posix_1fadvise( PASS_EXCEPTIONS(env); int err = 0; - if ((err = posix_fadvise(fd, (off_t)offset, (off_t)len, flags))) { + if ((err = posix_fadvise(fd, (off_t)offset, (off_t)len, map_fadvise_flag(flags)))) { #ifdef __FreeBSD__ throw_ioe(env, errno); #else From e0a9e1bfb724b60af322666b8267c7a9121f84a7 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 13 Aug 2014 00:54:29 +0000 Subject: [PATCH 10/67] HADOOP-10281. Create a scheduler, which assigns schedulables a priority level. (Contributed by Chris Li) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617643 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/ipc/DecayRpcScheduler.java | 522 ++++++++++++++++++ .../hadoop/ipc/DecayRpcSchedulerMXBean.java | 30 + .../org/apache/hadoop/ipc/RpcScheduler.java | 29 + .../hadoop/ipc/TestDecayRpcScheduler.java | 225 ++++++++ 5 files changed, 809 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 38bef36de79..06e9c25e35c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -505,6 +505,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10820. Throw an exception in GenericOptionsParser when passed an empty Path. (Alex Holmes and Zhihai Xu via wang) + HADOOP-10281. Create a scheduler, which assigns schedulables a priority + level. (Chris Li via Arpit Agarwal) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java new file mode 100644 index 00000000000..d06b25cbbaf --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java @@ -0,0 +1,522 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import java.lang.ref.WeakReference; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.metrics2.util.MBeans; + +import org.codehaus.jackson.map.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; + +/** + * The decay RPC scheduler counts incoming requests in a map, then + * decays the counts at a fixed time interval. The scheduler is optimized + * for large periods (on the order of seconds), as it offloads work to the + * decay sweep. + */ +public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean { + /** + * Period controls how many milliseconds between each decay sweep. + */ + public static final String IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY = + "faircallqueue.decay-scheduler.period-ms"; + public static final long IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT = + 5000L; + + /** + * Decay factor controls how much each count is suppressed by on each sweep. + * Valid numbers are > 0 and < 1. Decay factor works in tandem with period + * to control how long the scheduler remembers an identity. + */ + public static final String IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY = + "faircallqueue.decay-scheduler.decay-factor"; + public static final double IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT = + 0.5; + + /** + * Thresholds are specified as integer percentages, and specify which usage + * range each queue will be allocated to. For instance, specifying the list + * 10, 40, 80 + * implies 4 queues, with + * - q3 from 80% up + * - q2 from 40 up to 80 + * - q1 from 10 up to 40 + * - q0 otherwise. + */ + public static final String IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY = + "faircallqueue.decay-scheduler.thresholds"; + + // Specifies the identity to use when the IdentityProvider cannot handle + // a schedulable. + public static final String DECAYSCHEDULER_UNKNOWN_IDENTITY = + "IdentityProvider.Unknown"; + + public static final Log LOG = LogFactory.getLog(DecayRpcScheduler.class); + + // Track the number of calls for each schedulable identity + private final ConcurrentHashMap callCounts = + new ConcurrentHashMap(); + + // Should be the sum of all AtomicLongs in callCounts + private final AtomicLong totalCalls = new AtomicLong(); + + // Pre-computed scheduling decisions during the decay sweep are + // atomically swapped in as a read-only map + private final AtomicReference> scheduleCacheRef = + new AtomicReference>(); + + // Tune the behavior of the scheduler + private final long decayPeriodMillis; // How long between each tick + private final double decayFactor; // nextCount = currentCount / decayFactor + private final int numQueues; // affects scheduling decisions, from 0 to numQueues - 1 + private final double[] thresholds; + private final IdentityProvider identityProvider; + + /** + * This TimerTask will call decayCurrentCounts until + * the scheduler has been garbage collected. + */ + public static class DecayTask extends TimerTask { + private WeakReference schedulerRef; + private Timer timer; + + public DecayTask(DecayRpcScheduler scheduler, Timer timer) { + this.schedulerRef = new WeakReference(scheduler); + this.timer = timer; + } + + @Override + public void run() { + DecayRpcScheduler sched = schedulerRef.get(); + if (sched != null) { + sched.decayCurrentCounts(); + } else { + // Our scheduler was garbage collected since it is no longer in use, + // so we should terminate the timer as well + timer.cancel(); + timer.purge(); + } + } + } + + /** + * Create a decay scheduler. + * @param numQueues number of queues to schedule for + * @param ns config prefix, so that we can configure multiple schedulers + * in a single instance. + * @param conf configuration to use. + */ + public DecayRpcScheduler(int numQueues, String ns, Configuration conf) { + if (numQueues < 1) { + throw new IllegalArgumentException("number of queues must be > 0"); + } + + this.numQueues = numQueues; + this.decayFactor = parseDecayFactor(ns, conf); + this.decayPeriodMillis = parseDecayPeriodMillis(ns, conf); + this.identityProvider = this.parseIdentityProvider(ns, conf); + this.thresholds = parseThresholds(ns, conf, numQueues); + + // Setup delay timer + Timer timer = new Timer(); + DecayTask task = new DecayTask(this, timer); + timer.scheduleAtFixedRate(task, 0, this.decayPeriodMillis); + + MetricsProxy prox = MetricsProxy.getInstance(ns); + prox.setDelegate(this); + } + + // Load configs + private IdentityProvider parseIdentityProvider(String ns, Configuration conf) { + List providers = conf.getInstances( + ns + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY, + IdentityProvider.class); + + if (providers.size() < 1) { + LOG.info("IdentityProvider not specified, " + + "defaulting to UserIdentityProvider"); + return new UserIdentityProvider(); + } + + return providers.get(0); // use the first + } + + private static double parseDecayFactor(String ns, Configuration conf) { + double factor = conf.getDouble(ns + "." + + IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, + IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT + ); + + if (factor <= 0 || factor >= 1) { + throw new IllegalArgumentException("Decay Factor " + + "must be between 0 and 1"); + } + + return factor; + } + + private static long parseDecayPeriodMillis(String ns, Configuration conf) { + long period = conf.getLong(ns + "." + + IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, + IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT + ); + + if (period <= 0) { + throw new IllegalArgumentException("Period millis must be >= 0"); + } + + return period; + } + + private static double[] parseThresholds(String ns, Configuration conf, + int numQueues) { + int[] percentages = conf.getInts(ns + "." + + IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY); + + if (percentages.length == 0) { + return getDefaultThresholds(numQueues); + } else if (percentages.length != numQueues-1) { + throw new IllegalArgumentException("Number of thresholds should be " + + (numQueues-1) + ". Was: " + percentages.length); + } + + // Convert integer percentages to decimals + double[] decimals = new double[percentages.length]; + for (int i = 0; i < percentages.length; i++) { + decimals[i] = percentages[i] / 100.0; + } + + return decimals; + } + + /** + * Generate default thresholds if user did not specify. Strategy is + * to halve each time, since queue usage tends to be exponential. + * So if numQueues is 4, we would generate: double[]{0.125, 0.25, 0.5} + * which specifies the boundaries between each queue's usage. + * @param numQueues number of queues to compute for + * @return array of boundaries of length numQueues - 1 + */ + private static double[] getDefaultThresholds(int numQueues) { + double[] ret = new double[numQueues - 1]; + double div = Math.pow(2, numQueues - 1); + + for (int i = 0; i < ret.length; i++) { + ret[i] = Math.pow(2, i)/div; + } + return ret; + } + + /** + * Decay the stored counts for each user and clean as necessary. + * This method should be called periodically in order to keep + * counts current. + */ + private void decayCurrentCounts() { + long total = 0; + Iterator> it = + callCounts.entrySet().iterator(); + + while (it.hasNext()) { + Map.Entry entry = it.next(); + AtomicLong count = entry.getValue(); + + // Compute the next value by reducing it by the decayFactor + long currentValue = count.get(); + long nextValue = (long)(currentValue * decayFactor); + total += nextValue; + count.set(nextValue); + + if (nextValue == 0) { + // We will clean up unused keys here. An interesting optimization might + // be to have an upper bound on keyspace in callCounts and only + // clean once we pass it. + it.remove(); + } + } + + // Update the total so that we remain in sync + totalCalls.set(total); + + // Now refresh the cache of scheduling decisions + recomputeScheduleCache(); + } + + /** + * Update the scheduleCache to match current conditions in callCounts. + */ + private void recomputeScheduleCache() { + Map nextCache = new HashMap(); + + for (Map.Entry entry : callCounts.entrySet()) { + Object id = entry.getKey(); + AtomicLong value = entry.getValue(); + + long snapshot = value.get(); + int computedLevel = computePriorityLevel(snapshot); + + nextCache.put(id, computedLevel); + } + + // Swap in to activate + scheduleCacheRef.set(Collections.unmodifiableMap(nextCache)); + } + + /** + * Get the number of occurrences and increment atomically. + * @param identity the identity of the user to increment + * @return the value before incrementation + */ + private long getAndIncrement(Object identity) throws InterruptedException { + // We will increment the count, or create it if no such count exists + AtomicLong count = this.callCounts.get(identity); + if (count == null) { + // Create the count since no such count exists. + count = new AtomicLong(0); + + // Put it in, or get the AtomicInteger that was put in by another thread + AtomicLong otherCount = callCounts.putIfAbsent(identity, count); + if (otherCount != null) { + count = otherCount; + } + } + + // Update the total + totalCalls.getAndIncrement(); + + // At this point value is guaranteed to be not null. It may however have + // been clobbered from callCounts. Nonetheless, we return what + // we have. + return count.getAndIncrement(); + } + + /** + * Given the number of occurrences, compute a scheduling decision. + * @param occurrences how many occurrences + * @return scheduling decision from 0 to numQueues - 1 + */ + private int computePriorityLevel(long occurrences) { + long totalCallSnapshot = totalCalls.get(); + + double proportion = 0; + if (totalCallSnapshot > 0) { + proportion = (double) occurrences / totalCallSnapshot; + } + + // Start with low priority queues, since they will be most common + for(int i = (numQueues - 1); i > 0; i--) { + if (proportion >= this.thresholds[i - 1]) { + return i; // We've found our queue number + } + } + + // If we get this far, we're at queue 0 + return 0; + } + + /** + * Returns the priority level for a given identity by first trying the cache, + * then computing it. + * @param identity an object responding to toString and hashCode + * @return integer scheduling decision from 0 to numQueues - 1 + */ + private int cachedOrComputedPriorityLevel(Object identity) { + try { + long occurrences = this.getAndIncrement(identity); + + // Try the cache + Map scheduleCache = scheduleCacheRef.get(); + if (scheduleCache != null) { + Integer priority = scheduleCache.get(identity); + if (priority != null) { + return priority; + } + } + + // Cache was no good, compute it + return computePriorityLevel(occurrences); + } catch (InterruptedException ie) { + LOG.warn("Caught InterruptedException, returning low priority queue"); + return numQueues - 1; + } + } + + /** + * Compute the appropriate priority for a schedulable based on past requests. + * @param obj the schedulable obj to query and remember + * @return the queue index which we recommend scheduling in + */ + @Override + public int getPriorityLevel(Schedulable obj) { + // First get the identity + String identity = this.identityProvider.makeIdentity(obj); + if (identity == null) { + // Identity provider did not handle this + identity = DECAYSCHEDULER_UNKNOWN_IDENTITY; + } + + return cachedOrComputedPriorityLevel(identity); + } + + // For testing + @VisibleForTesting + public double getDecayFactor() { return decayFactor; } + + @VisibleForTesting + public long getDecayPeriodMillis() { return decayPeriodMillis; } + + @VisibleForTesting + public double[] getThresholds() { return thresholds; } + + @VisibleForTesting + public void forceDecay() { decayCurrentCounts(); } + + @VisibleForTesting + public Map getCallCountSnapshot() { + HashMap snapshot = new HashMap(); + + for (Map.Entry entry : callCounts.entrySet()) { + snapshot.put(entry.getKey(), entry.getValue().get()); + } + + return Collections.unmodifiableMap(snapshot); + } + + @VisibleForTesting + public long getTotalCallSnapshot() { + return totalCalls.get(); + } + + /** + * MetricsProxy is a singleton because we may init multiple schedulers and we + * want to clean up resources when a new scheduler replaces the old one. + */ + private static final class MetricsProxy implements DecayRpcSchedulerMXBean { + // One singleton per namespace + private static final HashMap INSTANCES = + new HashMap(); + + // Weakref for delegate, so we don't retain it forever if it can be GC'd + private WeakReference delegate; + + private MetricsProxy(String namespace) { + MBeans.register(namespace, "DecayRpcScheduler", this); + } + + public static synchronized MetricsProxy getInstance(String namespace) { + MetricsProxy mp = INSTANCES.get(namespace); + if (mp == null) { + // We must create one + mp = new MetricsProxy(namespace); + INSTANCES.put(namespace, mp); + } + return mp; + } + + public void setDelegate(DecayRpcScheduler obj) { + this.delegate = new WeakReference(obj); + } + + @Override + public String getSchedulingDecisionSummary() { + DecayRpcScheduler scheduler = delegate.get(); + if (scheduler == null) { + return "No Active Scheduler"; + } else { + return scheduler.getSchedulingDecisionSummary(); + } + } + + @Override + public String getCallVolumeSummary() { + DecayRpcScheduler scheduler = delegate.get(); + if (scheduler == null) { + return "No Active Scheduler"; + } else { + return scheduler.getCallVolumeSummary(); + } + } + + @Override + public int getUniqueIdentityCount() { + DecayRpcScheduler scheduler = delegate.get(); + if (scheduler == null) { + return -1; + } else { + return scheduler.getUniqueIdentityCount(); + } + } + + @Override + public long getTotalCallVolume() { + DecayRpcScheduler scheduler = delegate.get(); + if (scheduler == null) { + return -1; + } else { + return scheduler.getTotalCallVolume(); + } + } + } + + public int getUniqueIdentityCount() { + return callCounts.size(); + } + + public long getTotalCallVolume() { + return totalCalls.get(); + } + + public String getSchedulingDecisionSummary() { + Map decisions = scheduleCacheRef.get(); + if (decisions == null) { + return "{}"; + } else { + try { + ObjectMapper om = new ObjectMapper(); + return om.writeValueAsString(decisions); + } catch (Exception e) { + return "Error: " + e.getMessage(); + } + } + } + + public String getCallVolumeSummary() { + try { + ObjectMapper om = new ObjectMapper(); + return om.writeValueAsString(callCounts); + } catch (Exception e) { + return "Error: " + e.getMessage(); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java new file mode 100644 index 00000000000..3481f19449d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +/** + * Provides metrics for Decay scheduler. + */ +public interface DecayRpcSchedulerMXBean { + // Get an overview of the requests in history. + String getSchedulingDecisionSummary(); + String getCallVolumeSummary(); + int getUniqueIdentityCount(); + long getTotalCallVolume(); +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java new file mode 100644 index 00000000000..a1557061809 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +/** + * Implement this interface to be used for RPC scheduling in the fair call queues. + */ +public interface RpcScheduler { + /** + * Returns priority level greater than zero as a hint for scheduling. + */ + int getPriorityLevel(Schedulable obj); +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java new file mode 100644 index 00000000000..edc3b0051ab --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java @@ -0,0 +1,225 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.Arrays; +import org.junit.Test; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.fs.CommonConfigurationKeys; + +public class TestDecayRpcScheduler { + private Schedulable mockCall(String id) { + Schedulable mockCall = mock(Schedulable.class); + UserGroupInformation ugi = mock(UserGroupInformation.class); + + when(ugi.getUserName()).thenReturn(id); + when(mockCall.getUserGroupInformation()).thenReturn(ugi); + + return mockCall; + } + + private DecayRpcScheduler scheduler; + + @Test(expected=IllegalArgumentException.class) + public void testNegativeScheduler() { + scheduler = new DecayRpcScheduler(-1, "", new Configuration()); + } + + @Test(expected=IllegalArgumentException.class) + public void testZeroScheduler() { + scheduler = new DecayRpcScheduler(0, "", new Configuration()); + } + + @Test + public void testParsePeriod() { + // By default + scheduler = new DecayRpcScheduler(1, "", new Configuration()); + assertEquals(DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT, + scheduler.getDecayPeriodMillis()); + + // Custom + Configuration conf = new Configuration(); + conf.setLong("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, + 1058); + scheduler = new DecayRpcScheduler(1, "ns", conf); + assertEquals(1058L, scheduler.getDecayPeriodMillis()); + } + + @Test + public void testParseFactor() { + // Default + scheduler = new DecayRpcScheduler(1, "", new Configuration()); + assertEquals(DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT, + scheduler.getDecayFactor(), 0.00001); + + // Custom + Configuration conf = new Configuration(); + conf.set("prefix." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, + "0.125"); + scheduler = new DecayRpcScheduler(1, "prefix", conf); + assertEquals(0.125, scheduler.getDecayFactor(), 0.00001); + } + + public void assertEqualDecimalArrays(double[] a, double[] b) { + assertEquals(a.length, b.length); + for(int i = 0; i < a.length; i++) { + assertEquals(a[i], b[i], 0.00001); + } + } + + @Test + public void testParseThresholds() { + // Defaults vary by number of queues + Configuration conf = new Configuration(); + scheduler = new DecayRpcScheduler(1, "", conf); + assertEqualDecimalArrays(new double[]{}, scheduler.getThresholds()); + + scheduler = new DecayRpcScheduler(2, "", conf); + assertEqualDecimalArrays(new double[]{0.5}, scheduler.getThresholds()); + + scheduler = new DecayRpcScheduler(3, "", conf); + assertEqualDecimalArrays(new double[]{0.25, 0.5}, scheduler.getThresholds()); + + scheduler = new DecayRpcScheduler(4, "", conf); + assertEqualDecimalArrays(new double[]{0.125, 0.25, 0.5}, scheduler.getThresholds()); + + // Custom + conf = new Configuration(); + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY, + "1, 10, 20, 50, 85"); + scheduler = new DecayRpcScheduler(6, "ns", conf); + assertEqualDecimalArrays(new double[]{0.01, 0.1, 0.2, 0.5, 0.85}, scheduler.getThresholds()); + } + + @Test + public void testAccumulate() { + Configuration conf = new Configuration(); + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush + scheduler = new DecayRpcScheduler(1, "ns", conf); + + assertEquals(0, scheduler.getCallCountSnapshot().size()); // empty first + + scheduler.getPriorityLevel(mockCall("A")); + assertEquals(1, scheduler.getCallCountSnapshot().get("A").longValue()); + assertEquals(1, scheduler.getCallCountSnapshot().get("A").longValue()); + + scheduler.getPriorityLevel(mockCall("A")); + scheduler.getPriorityLevel(mockCall("B")); + scheduler.getPriorityLevel(mockCall("A")); + + assertEquals(3, scheduler.getCallCountSnapshot().get("A").longValue()); + assertEquals(1, scheduler.getCallCountSnapshot().get("B").longValue()); + } + + @Test + public void testDecay() { + Configuration conf = new Configuration(); + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "999999999"); // Never + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, "0.5"); + scheduler = new DecayRpcScheduler(1, "ns", conf); + + assertEquals(0, scheduler.getTotalCallSnapshot()); + + for (int i = 0; i < 4; i++) { + scheduler.getPriorityLevel(mockCall("A")); + } + + for (int i = 0; i < 8; i++) { + scheduler.getPriorityLevel(mockCall("B")); + } + + assertEquals(12, scheduler.getTotalCallSnapshot()); + assertEquals(4, scheduler.getCallCountSnapshot().get("A").longValue()); + assertEquals(8, scheduler.getCallCountSnapshot().get("B").longValue()); + + scheduler.forceDecay(); + + assertEquals(6, scheduler.getTotalCallSnapshot()); + assertEquals(2, scheduler.getCallCountSnapshot().get("A").longValue()); + assertEquals(4, scheduler.getCallCountSnapshot().get("B").longValue()); + + scheduler.forceDecay(); + + assertEquals(3, scheduler.getTotalCallSnapshot()); + assertEquals(1, scheduler.getCallCountSnapshot().get("A").longValue()); + assertEquals(2, scheduler.getCallCountSnapshot().get("B").longValue()); + + scheduler.forceDecay(); + + assertEquals(1, scheduler.getTotalCallSnapshot()); + assertEquals(null, scheduler.getCallCountSnapshot().get("A")); + assertEquals(1, scheduler.getCallCountSnapshot().get("B").longValue()); + + scheduler.forceDecay(); + + assertEquals(0, scheduler.getTotalCallSnapshot()); + assertEquals(null, scheduler.getCallCountSnapshot().get("A")); + assertEquals(null, scheduler.getCallCountSnapshot().get("B")); + } + + @Test + public void testPriority() { + Configuration conf = new Configuration(); + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY, + "25, 50, 75"); + scheduler = new DecayRpcScheduler(4, "ns", conf); + + assertEquals(0, scheduler.getPriorityLevel(mockCall("A"))); + assertEquals(2, scheduler.getPriorityLevel(mockCall("A"))); + assertEquals(0, scheduler.getPriorityLevel(mockCall("B"))); + assertEquals(1, scheduler.getPriorityLevel(mockCall("B"))); + assertEquals(0, scheduler.getPriorityLevel(mockCall("C"))); + assertEquals(0, scheduler.getPriorityLevel(mockCall("C"))); + assertEquals(1, scheduler.getPriorityLevel(mockCall("A"))); + assertEquals(1, scheduler.getPriorityLevel(mockCall("A"))); + assertEquals(1, scheduler.getPriorityLevel(mockCall("A"))); + assertEquals(2, scheduler.getPriorityLevel(mockCall("A"))); + } + + @Test(timeout=2000) + public void testPeriodic() throws InterruptedException { + Configuration conf = new Configuration(); + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "10"); + conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, "0.5"); + scheduler = new DecayRpcScheduler(1, "ns", conf); + + assertEquals(10, scheduler.getDecayPeriodMillis()); + assertEquals(0, scheduler.getTotalCallSnapshot()); + + for (int i = 0; i < 64; i++) { + scheduler.getPriorityLevel(mockCall("A")); + } + + // It should eventually decay to zero + while (scheduler.getTotalCallSnapshot() > 0) { + Thread.sleep(10); + } + } +} \ No newline at end of file From 5197f8c3c53f18701f041dd2301b454930a5025b Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Wed, 13 Aug 2014 01:38:59 +0000 Subject: [PATCH 11/67] YARN-1370. Fair scheduler to re-populate container allocation state. (Anubhav Dhoot via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617645 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../resourcemanager/scheduler/fair/FairScheduler.java | 2 ++ .../resourcemanager/TestWorkPreservingRMRestart.java | 9 +++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d977127cd9d..55a61126fe2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -125,6 +125,9 @@ Release 2.6.0 - UNRELEASED YARN-2399. FairScheduler: Merge AppSchedulable and FSSchedulerApp into FSAppAttempt. (kasha) + YARN-1370. Fair scheduler to re-populate container allocation state. + (Anubhav Dhoot via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 4748d73e038..aa3f8240947 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1150,6 +1150,8 @@ public class FairScheduler extends } NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event; addNode(nodeAddedEvent.getAddedRMNode()); + recoverContainersOnNode(nodeAddedEvent.getContainerReports(), + nodeAddedEvent.getAddedRMNode()); break; case NODE_REMOVED: if (!(event instanceof NodeRemovedSchedulerEvent)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 24a2f437953..9cae07c7fb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -107,7 +108,7 @@ public class TestWorkPreservingRMRestart { @Parameterized.Parameters public static Collection getTestParameters() { return Arrays.asList(new Object[][] { { CapacityScheduler.class }, - { FifoScheduler.class } }); + { FifoScheduler.class }, {FairScheduler.class } }); } public TestWorkPreservingRMRestart(Class schedulerClass) { @@ -224,7 +225,11 @@ public class TestWorkPreservingRMRestart { assertTrue(schedulerAttempt.getLiveContainers().contains( scheduler.getRMContainer(runningContainer.getContainerId()))); assertEquals(schedulerAttempt.getCurrentConsumption(), usedResources); - assertEquals(availableResources, schedulerAttempt.getHeadroom()); + + // Until YARN-1959 is resolved + if (scheduler.getClass() != FairScheduler.class) { + assertEquals(availableResources, schedulerAttempt.getHeadroom()); + } // *********** check appSchedulingInfo state *********** assertEquals((1 << 22) + 1, schedulerAttempt.getNewContainerId()); From a34dafe325902164c04de0bf9d34eccdf6a2cd88 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 13 Aug 2014 17:30:26 +0000 Subject: [PATCH 12/67] HADOOP-8944. Shell command fs -count should include human readable option (Jonathan Allen via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617775 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/fs/ContentSummary.java | 48 +++++++++++++++---- .../org/apache/hadoop/fs/shell/Count.java | 36 ++++++++++++-- .../src/site/apt/FileSystemShell.apt.vm | 6 ++- .../src/test/resources/testConf.xml | 6 ++- .../src/test/resources/testHDFSConf.xml | 44 +++++++++++++++++ 6 files changed, 126 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 06e9c25e35c..547f2318feb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -202,6 +202,9 @@ Trunk (Unreleased) HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting underlying store. (asuresh via tucu) + HADOOP-8944. Shell command fs -count should include human readable option + (Jonathan Allen via aw) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index 0d685b43e1f..5d01637a0fb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.StringUtils; /** Store the summary of a content (a directory or a file). */ @InterfaceAudience.Public @@ -102,7 +103,7 @@ public class ContentSummary implements Writable{ * <----12----> <----12----> <-------18-------> * DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME */ - private static final String STRING_FORMAT = "%12d %12d %18d "; + private static final String STRING_FORMAT = "%12s %12s %18s "; /** * Output format: * <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18-------> @@ -117,7 +118,7 @@ public class ContentSummary implements Writable{ private static final String QUOTA_HEADER = String.format( QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, - "quota", "remaining quota", "space quota", "reamaining quota") + + "name quota", "rem name quota", "space quota", "rem space quota") + HEADER; /** Return the header of the output. @@ -139,11 +140,25 @@ public class ContentSummary implements Writable{ /** Return the string representation of the object in the output format. * if qOption is false, output directory count, file count, and content size; * if qOption is true, output quota and remaining quota as well. - * + * * @param qOption a flag indicating if quota needs to be printed or not * @return the string representation of the object - */ + */ public String toString(boolean qOption) { + return toString(qOption, false); + } + + /** Return the string representation of the object in the output format. + * if qOption is false, output directory count, file count, and content size; + * if qOption is true, output quota and remaining quota as well. + * if hOption is false file sizes are returned in bytes + * if hOption is true file sizes are returned in human readable + * + * @param qOption a flag indicating if quota needs to be printed or not + * @param hOption a flag indicating if human readable output if to be used + * @return the string representation of the object + */ + public String toString(boolean qOption, boolean hOption) { String prefix = ""; if (qOption) { String quotaStr = "none"; @@ -152,19 +167,32 @@ public class ContentSummary implements Writable{ String spaceQuotaRem = "inf"; if (quota>0) { - quotaStr = Long.toString(quota); - quotaRem = Long.toString(quota-(directoryCount+fileCount)); + quotaStr = formatSize(quota, hOption); + quotaRem = formatSize(quota-(directoryCount+fileCount), hOption); } if (spaceQuota>0) { - spaceQuotaStr = Long.toString(spaceQuota); - spaceQuotaRem = Long.toString(spaceQuota - spaceConsumed); + spaceQuotaStr = formatSize(spaceQuota, hOption); + spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption); } prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem); } - return prefix + String.format(STRING_FORMAT, directoryCount, - fileCount, length); + return prefix + String.format(STRING_FORMAT, + formatSize(directoryCount, hOption), + formatSize(fileCount, hOption), + formatSize(length, hOption)); + } + /** + * Formats a size to be human readable or in bytes + * @param size value to be formatted + * @param humanReadable flag indicating human readable or not + * @return String representation of the size + */ + private String formatSize(long size, boolean humanReadable) { + return humanReadable + ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1) + : String.valueOf(size); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index b8ccc0c9ec9..ff7a10f2975 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -42,16 +42,22 @@ public class Count extends FsCommand { factory.addClass(Count.class, "-count"); } + private static final String OPTION_QUOTA = "q"; + private static final String OPTION_HUMAN = "h"; + public static final String NAME = "count"; - public static final String USAGE = "[-q] ..."; + public static final String USAGE = + "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] ..."; public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + "that match the specified file pattern. The output columns are:\n" + "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" + "QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" + - " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME"; + " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" + + "The -h option shows file sizes in human readable format."; private boolean showQuotas; + private boolean humanReadable; /** Constructor */ public Count() {} @@ -70,17 +76,37 @@ public class Count extends FsCommand { @Override protected void processOptions(LinkedList args) { - CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "q"); + CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, + OPTION_QUOTA, OPTION_HUMAN); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory args.add("."); } - showQuotas = cf.getOpt("q"); + showQuotas = cf.getOpt(OPTION_QUOTA); + humanReadable = cf.getOpt(OPTION_HUMAN); } @Override protected void processPath(PathData src) throws IOException { ContentSummary summary = src.fs.getContentSummary(src.path); - out.println(summary.toString(showQuotas) + src); + out.println(summary.toString(showQuotas, isHumanReadable()) + src); + } + + /** + * Should quotas get shown as part of the report? + * @return if quotas should be shown then true otherwise false + */ + @InterfaceAudience.Private + boolean isShowQuotas() { + return showQuotas; + } + + /** + * Should sizes be shown in human readable format rather than bytes? + * @return true if human readable format + */ + @InterfaceAudience.Private + boolean isHumanReadable() { + return humanReadable; } } diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm index 4a82884fb37..48e0b21220f 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm @@ -138,7 +138,7 @@ copyToLocal count - Usage: << >>> + Usage: << >>> Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR_COUNT, @@ -147,12 +147,16 @@ count The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA, REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME + The -h option shows sizes in human readable format. + Example: * <<>> * <<>> + * <<>> + Exit Code: Returns 0 on success and -1 on error. diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index a91f0416d67..c44eddafe43 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -238,7 +238,7 @@ RegexpComparator - ^-count \[-q\] <path> \.\.\. :\s* + ^-count \[-q\] \[-h\] <path> \.\.\. :( )* RegexpComparator @@ -260,6 +260,10 @@ RegexpComparator ^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )* + + RegexpComparator + ^( |\t)*The -h option shows file sizes in human readable format.( )* + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 03083c50cea..087c3ab48aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -8655,6 +8655,50 @@ + + count: file using -h option + + -fs NAMENODE -mkdir -p dir + -fs NAMENODE -put CLITEST_DATA/data15bytes file1 + -fs NAMENODE -put CLITEST_DATA/data1k file2 + -fs NAMENODE -count -h file1 file2 + + + -fs NAMENODE -rm file1 file2 + + + + RegexpComparator + ( |\t)*0( |\t)*1( |\t)*15 file1 + + + + + RegexpComparator + ( |\t)*0( |\t)*1( |\t)*1\.0 K file2 + + + + + + count: directory using -q and -h options + + -fs NAMENODE -mkdir /dir1 + -fs NAMENODE -setQuota 10 /dir1 + -fs NAMENODE -setSpaceQuota 1m /dir1 + -fs NAMENODE -count -q -h /dir1 + + + -fs NAMENODE -rm -r /dir1 + + + + RegexpComparator + ( |\t)*10( |\t)*9( |\t)*1 M( |\t)*1 M( |\t)*1( |\t)*0( |\t)*0 /dir1 + + + + chmod: change permission(octal mode) of file in absolute path From 03d8ff496a6fca15eca123d081361594bb47e0d8 Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Wed, 13 Aug 2014 17:45:47 +0000 Subject: [PATCH 13/67] HDFS-6567. Normalize the order of public final in HdfsFileStatus. Contributed by Tassapol Athiapinya. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617779 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/protocol/HdfsFileStatus.java | 38 +++++++++---------- .../hdfs/protocol/HdfsLocatedFileStatus.java | 2 +- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8148052005b..685cd6211d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -396,6 +396,9 @@ Release 2.6.0 - UNRELEASED HDFS-6836. HDFS INFO logging is verbose & uses file appenders. (Xiaoyu Yao via Arpit Agarwal) + HDFS-6567. Normalize the order of public final in HdfsFileStatus. + (Tassapol Athiapinya via wheat9) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index 66c56faa14b..0652de14aee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -91,7 +91,7 @@ public class HdfsFileStatus { * Get the length of this file, in bytes. * @return the length of this file, in bytes. */ - final public long getLen() { + public final long getLen() { return length; } @@ -99,7 +99,7 @@ public class HdfsFileStatus { * Is this a directory? * @return true if this is a directory */ - final public boolean isDir() { + public final boolean isDir() { return isdir; } @@ -115,7 +115,7 @@ public class HdfsFileStatus { * Get the block size of the file. * @return the number of bytes */ - final public long getBlockSize() { + public final long getBlockSize() { return blocksize; } @@ -123,7 +123,7 @@ public class HdfsFileStatus { * Get the replication factor of a file. * @return the replication factor of a file. */ - final public short getReplication() { + public final short getReplication() { return block_replication; } @@ -131,7 +131,7 @@ public class HdfsFileStatus { * Get the modification time of the file. * @return the modification time of file in milliseconds since January 1, 1970 UTC. */ - final public long getModificationTime() { + public final long getModificationTime() { return modification_time; } @@ -139,7 +139,7 @@ public class HdfsFileStatus { * Get the access time of the file. * @return the access time of file in milliseconds since January 1, 1970 UTC. */ - final public long getAccessTime() { + public final long getAccessTime() { return access_time; } @@ -147,7 +147,7 @@ public class HdfsFileStatus { * Get FsPermission associated with the file. * @return permssion */ - final public FsPermission getPermission() { + public final FsPermission getPermission() { return permission; } @@ -155,7 +155,7 @@ public class HdfsFileStatus { * Get the owner of the file. * @return owner of the file */ - final public String getOwner() { + public final String getOwner() { return owner; } @@ -163,7 +163,7 @@ public class HdfsFileStatus { * Get the group associated with the file. * @return group for the file. */ - final public String getGroup() { + public final String getGroup() { return group; } @@ -171,7 +171,7 @@ public class HdfsFileStatus { * Check if the local name is empty * @return true if the name is empty */ - final public boolean isEmptyLocalName() { + public final boolean isEmptyLocalName() { return path.length == 0; } @@ -179,7 +179,7 @@ public class HdfsFileStatus { * Get the string representation of the local name * @return the local name in string */ - final public String getLocalName() { + public final String getLocalName() { return DFSUtil.bytes2String(path); } @@ -187,7 +187,7 @@ public class HdfsFileStatus { * Get the Java UTF8 representation of the local name * @return the local name in java UTF8 */ - final public byte[] getLocalNameInBytes() { + public final byte[] getLocalNameInBytes() { return path; } @@ -196,7 +196,7 @@ public class HdfsFileStatus { * @param parent the parent path * @return the full path in string */ - final public String getFullName(final String parent) { + public final String getFullName(final String parent) { if (isEmptyLocalName()) { return parent; } @@ -214,7 +214,7 @@ public class HdfsFileStatus { * @param parent the parent path * @return the full path */ - final public Path getFullPath(final Path parent) { + public final Path getFullPath(final Path parent) { if (isEmptyLocalName()) { return parent; } @@ -226,23 +226,23 @@ public class HdfsFileStatus { * Get the string representation of the symlink. * @return the symlink as a string. */ - final public String getSymlink() { + public final String getSymlink() { return DFSUtil.bytes2String(symlink); } - final public byte[] getSymlinkInBytes() { + public final byte[] getSymlinkInBytes() { return symlink; } - final public long getFileId() { + public final long getFileId() { return fileId; } - final public int getChildrenNum() { + public final int getChildrenNum() { return childrenNum; } - final public FileStatus makeQualified(URI defaultUri, Path path) { + public final FileStatus makeQualified(URI defaultUri, Path path) { return new FileStatus(getLen(), isDir(), getReplication(), getBlockSize(), getModificationTime(), getAccessTime(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java index 0f90e435a43..6199b8eb0d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@ -67,7 +67,7 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus { return locations; } - final public LocatedFileStatus makeQualifiedLocated(URI defaultUri, + public final LocatedFileStatus makeQualifiedLocated(URI defaultUri, Path path) { return new LocatedFileStatus(getLen(), isDir(), getReplication(), getBlockSize(), getModificationTime(), From 471b1368e2a81b4d9850f0f4d98d31df1451354c Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Wed, 13 Aug 2014 18:06:33 +0000 Subject: [PATCH 14/67] HDFS-6847. Avoid timeouts for replaceBlock() call by sending intermediate responses to Balancer (Contributed by Vinayakumar B.) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617784 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/balancer/Dispatcher.java | 17 ++++------ .../hdfs/server/datanode/BlockReceiver.java | 32 ++++++++++++++++++- .../hdfs/server/datanode/DataXceiver.java | 9 +++--- .../src/main/proto/datatransfer.proto | 1 + .../server/datanode/TestBlockReplacement.java | 6 ++-- 6 files changed, 50 insertions(+), 18 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 685cd6211d6..d1ed85182e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -500,6 +500,9 @@ Release 2.6.0 - UNRELEASED HDFS-6830. BlockInfo.addStorage fails when DN changes the storage for a block replica (Arpit Agarwal) + HDFS-6847. Avoid timeouts for replaceBlock() call by sending intermediate + responses to Balancer (vinayakumarb) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index 4a6f96be7e5..b7dceb6f48d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -87,8 +87,6 @@ public class Dispatcher { private static final int MAX_NO_PENDING_MOVE_ITERATIONS = 5; private static final long DELAY_AFTER_ERROR = 10 * 1000L; // 10 seconds - private static final int BLOCK_MOVE_READ_TIMEOUT = 20 * 60 * 1000; // 20 - // minutes private final NameNodeConnector nnc; private final SaslDataTransferClient saslClient; @@ -278,13 +276,6 @@ public class Dispatcher { sock.connect( NetUtils.createSocketAddr(target.getDatanodeInfo().getXferAddr()), HdfsServerConstants.READ_TIMEOUT); - /* - * Unfortunately we don't have a good way to know if the Datanode is - * taking a really long time to move a block, OR something has gone - * wrong and it's never going to finish. To deal with this scenario, we - * set a long timeout (20 minutes) to avoid hanging indefinitely. - */ - sock.setSoTimeout(BLOCK_MOVE_READ_TIMEOUT); sock.setKeepAlive(true); @@ -341,8 +332,12 @@ public class Dispatcher { /** Receive a block copy response from the input stream */ private void receiveResponse(DataInputStream in) throws IOException { - BlockOpResponseProto response = BlockOpResponseProto - .parseFrom(vintPrefixed(in)); + BlockOpResponseProto response = + BlockOpResponseProto.parseFrom(vintPrefixed(in)); + while (response.getStatus() == Status.IN_PROGRESS) { + // read intermediate responses + response = BlockOpResponseProto.parseFrom(vintPrefixed(in)); + } if (response.getStatus() != Status.SUCCESS) { if (response.getStatus() == Status.ERROR_ACCESS_TOKEN) { throw new IOException("block move failed due to access token error"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index fb7ecd69e9a..9dcd006ed51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver; import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck; +import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; @@ -123,6 +124,14 @@ class BlockReceiver implements Closeable { private boolean syncOnClose; private long restartBudget; + /** + * for replaceBlock response + */ + private final long responseInterval; + private long lastResponseTime = 0; + private boolean isReplaceBlock = false; + private DataOutputStream replyOut = null; + BlockReceiver(final ExtendedBlock block, final StorageType storageType, final DataInputStream in, final String inAddr, final String myAddr, @@ -144,6 +153,9 @@ class BlockReceiver implements Closeable { this.isClient = !this.isDatanode; this.restartBudget = datanode.getDnConf().restartReplicaExpiry; this.datanodeSlowLogThresholdMs = datanode.getDnConf().datanodeSlowIoWarningThresholdMs; + // For replaceBlock() calls response should be sent to avoid socketTimeout + // at clients. So sending with the interval of 0.5 * socketTimeout + this.responseInterval = (long) (datanode.getDnConf().socketTimeout * 0.5); //for datanode, we have //1: clientName.length() == 0, and //2: stage == null or PIPELINE_SETUP_CREATE @@ -651,6 +663,20 @@ class BlockReceiver implements Closeable { lastPacketInBlock, offsetInBlock, Status.SUCCESS); } + /* + * Send in-progress responses for the replaceBlock() calls back to caller to + * avoid timeouts due to balancer throttling. HDFS-6247 + */ + if (isReplaceBlock + && (Time.monotonicNow() - lastResponseTime > responseInterval)) { + BlockOpResponseProto.Builder response = BlockOpResponseProto.newBuilder() + .setStatus(Status.IN_PROGRESS); + response.build().writeDelimitedTo(replyOut); + replyOut.flush(); + + lastResponseTime = Time.monotonicNow(); + } + if (throttler != null) { // throttle I/O throttler.throttle(len); } @@ -718,7 +744,8 @@ class BlockReceiver implements Closeable { DataInputStream mirrIn, // input from next datanode DataOutputStream replyOut, // output to previous datanode String mirrAddr, DataTransferThrottler throttlerArg, - DatanodeInfo[] downstreams) throws IOException { + DatanodeInfo[] downstreams, + boolean isReplaceBlock) throws IOException { syncOnClose = datanode.getDnConf().syncOnClose; boolean responderClosed = false; @@ -726,6 +753,9 @@ class BlockReceiver implements Closeable { mirrorAddr = mirrAddr; throttler = throttlerArg; + this.replyOut = replyOut; + this.isReplaceBlock = isReplaceBlock; + try { if (isClient && !isTransfer) { responder = new Daemon(datanode.threadGroup, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 5ef6cc7ee22..01fe036f1d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -708,7 +708,7 @@ class DataXceiver extends Receiver implements Runnable { if (blockReceiver != null) { String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, - mirrorAddr, null, targets); + mirrorAddr, null, targets, false); // send close-ack for transfer-RBW/Finalized if (isTransfer) { @@ -983,7 +983,7 @@ class DataXceiver extends Receiver implements Runnable { String errMsg = null; BlockReceiver blockReceiver = null; DataInputStream proxyReply = null; - + DataOutputStream replyOut = new DataOutputStream(getOutputStream()); try { // get the output stream to the proxy final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname); @@ -1040,8 +1040,8 @@ class DataXceiver extends Receiver implements Runnable { CachingStrategy.newDropBehind()); // receive a block - blockReceiver.receiveBlock(null, null, null, null, - dataXceiverServer.balanceThrottler, null); + blockReceiver.receiveBlock(null, null, replyOut, null, + dataXceiverServer.balanceThrottler, null, true); // notify name node datanode.notifyNamenodeReceivedBlock( @@ -1076,6 +1076,7 @@ class DataXceiver extends Receiver implements Runnable { IOUtils.closeStream(proxyOut); IOUtils.closeStream(blockReceiver); IOUtils.closeStream(proxyReply); + IOUtils.closeStream(replyOut); } //update metrics diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto index 9b4ba339d23..6283b569dda 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto @@ -207,6 +207,7 @@ enum Status { OOB_RESERVED1 = 9; // Reserved OOB_RESERVED2 = 10; // Reserved OOB_RESERVED3 = 11; // Reserved + IN_PROGRESS = 12; } message PipelineAckProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java index 478b6d1546b..e0d79648a88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java @@ -272,8 +272,10 @@ public class TestBlockReplacement { // receiveResponse DataInputStream reply = new DataInputStream(sock.getInputStream()); - BlockOpResponseProto proto = - BlockOpResponseProto.parseDelimitedFrom(reply); + BlockOpResponseProto proto = BlockOpResponseProto.parseDelimitedFrom(reply); + while (proto.getStatus() == Status.IN_PROGRESS) { + proto = BlockOpResponseProto.parseDelimitedFrom(reply); + } return proto.getStatus() == Status.SUCCESS; } From 6554994fab2d8a2a139fb71ed54be144f4057e08 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Wed, 13 Aug 2014 18:36:51 +0000 Subject: [PATCH 15/67] Reverted Merged revision(s) 1617784 from hadoop/common/trunk: HDFS-6847. Avoid timeouts for replaceBlock() call by sending intermediate responses to Balancer (Contributed by Vinayakumar B.) ........ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617794 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 -- .../hdfs/server/balancer/Dispatcher.java | 17 ++++++---- .../hdfs/server/datanode/BlockReceiver.java | 32 +------------------ .../hdfs/server/datanode/DataXceiver.java | 9 +++--- .../src/main/proto/datatransfer.proto | 1 - .../server/datanode/TestBlockReplacement.java | 6 ++-- 6 files changed, 18 insertions(+), 50 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d1ed85182e7..685cd6211d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -500,9 +500,6 @@ Release 2.6.0 - UNRELEASED HDFS-6830. BlockInfo.addStorage fails when DN changes the storage for a block replica (Arpit Agarwal) - HDFS-6847. Avoid timeouts for replaceBlock() call by sending intermediate - responses to Balancer (vinayakumarb) - Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index b7dceb6f48d..4a6f96be7e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -87,6 +87,8 @@ public class Dispatcher { private static final int MAX_NO_PENDING_MOVE_ITERATIONS = 5; private static final long DELAY_AFTER_ERROR = 10 * 1000L; // 10 seconds + private static final int BLOCK_MOVE_READ_TIMEOUT = 20 * 60 * 1000; // 20 + // minutes private final NameNodeConnector nnc; private final SaslDataTransferClient saslClient; @@ -276,6 +278,13 @@ public class Dispatcher { sock.connect( NetUtils.createSocketAddr(target.getDatanodeInfo().getXferAddr()), HdfsServerConstants.READ_TIMEOUT); + /* + * Unfortunately we don't have a good way to know if the Datanode is + * taking a really long time to move a block, OR something has gone + * wrong and it's never going to finish. To deal with this scenario, we + * set a long timeout (20 minutes) to avoid hanging indefinitely. + */ + sock.setSoTimeout(BLOCK_MOVE_READ_TIMEOUT); sock.setKeepAlive(true); @@ -332,12 +341,8 @@ public class Dispatcher { /** Receive a block copy response from the input stream */ private void receiveResponse(DataInputStream in) throws IOException { - BlockOpResponseProto response = - BlockOpResponseProto.parseFrom(vintPrefixed(in)); - while (response.getStatus() == Status.IN_PROGRESS) { - // read intermediate responses - response = BlockOpResponseProto.parseFrom(vintPrefixed(in)); - } + BlockOpResponseProto response = BlockOpResponseProto + .parseFrom(vintPrefixed(in)); if (response.getStatus() != Status.SUCCESS) { if (response.getStatus() == Status.ERROR_ACCESS_TOKEN) { throw new IOException("block move failed due to access token error"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 9dcd006ed51..fb7ecd69e9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver; import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck; -import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; @@ -124,14 +123,6 @@ class BlockReceiver implements Closeable { private boolean syncOnClose; private long restartBudget; - /** - * for replaceBlock response - */ - private final long responseInterval; - private long lastResponseTime = 0; - private boolean isReplaceBlock = false; - private DataOutputStream replyOut = null; - BlockReceiver(final ExtendedBlock block, final StorageType storageType, final DataInputStream in, final String inAddr, final String myAddr, @@ -153,9 +144,6 @@ class BlockReceiver implements Closeable { this.isClient = !this.isDatanode; this.restartBudget = datanode.getDnConf().restartReplicaExpiry; this.datanodeSlowLogThresholdMs = datanode.getDnConf().datanodeSlowIoWarningThresholdMs; - // For replaceBlock() calls response should be sent to avoid socketTimeout - // at clients. So sending with the interval of 0.5 * socketTimeout - this.responseInterval = (long) (datanode.getDnConf().socketTimeout * 0.5); //for datanode, we have //1: clientName.length() == 0, and //2: stage == null or PIPELINE_SETUP_CREATE @@ -663,20 +651,6 @@ class BlockReceiver implements Closeable { lastPacketInBlock, offsetInBlock, Status.SUCCESS); } - /* - * Send in-progress responses for the replaceBlock() calls back to caller to - * avoid timeouts due to balancer throttling. HDFS-6247 - */ - if (isReplaceBlock - && (Time.monotonicNow() - lastResponseTime > responseInterval)) { - BlockOpResponseProto.Builder response = BlockOpResponseProto.newBuilder() - .setStatus(Status.IN_PROGRESS); - response.build().writeDelimitedTo(replyOut); - replyOut.flush(); - - lastResponseTime = Time.monotonicNow(); - } - if (throttler != null) { // throttle I/O throttler.throttle(len); } @@ -744,8 +718,7 @@ class BlockReceiver implements Closeable { DataInputStream mirrIn, // input from next datanode DataOutputStream replyOut, // output to previous datanode String mirrAddr, DataTransferThrottler throttlerArg, - DatanodeInfo[] downstreams, - boolean isReplaceBlock) throws IOException { + DatanodeInfo[] downstreams) throws IOException { syncOnClose = datanode.getDnConf().syncOnClose; boolean responderClosed = false; @@ -753,9 +726,6 @@ class BlockReceiver implements Closeable { mirrorAddr = mirrAddr; throttler = throttlerArg; - this.replyOut = replyOut; - this.isReplaceBlock = isReplaceBlock; - try { if (isClient && !isTransfer) { responder = new Daemon(datanode.threadGroup, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 01fe036f1d9..5ef6cc7ee22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -708,7 +708,7 @@ class DataXceiver extends Receiver implements Runnable { if (blockReceiver != null) { String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, - mirrorAddr, null, targets, false); + mirrorAddr, null, targets); // send close-ack for transfer-RBW/Finalized if (isTransfer) { @@ -983,7 +983,7 @@ class DataXceiver extends Receiver implements Runnable { String errMsg = null; BlockReceiver blockReceiver = null; DataInputStream proxyReply = null; - DataOutputStream replyOut = new DataOutputStream(getOutputStream()); + try { // get the output stream to the proxy final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname); @@ -1040,8 +1040,8 @@ class DataXceiver extends Receiver implements Runnable { CachingStrategy.newDropBehind()); // receive a block - blockReceiver.receiveBlock(null, null, replyOut, null, - dataXceiverServer.balanceThrottler, null, true); + blockReceiver.receiveBlock(null, null, null, null, + dataXceiverServer.balanceThrottler, null); // notify name node datanode.notifyNamenodeReceivedBlock( @@ -1076,7 +1076,6 @@ class DataXceiver extends Receiver implements Runnable { IOUtils.closeStream(proxyOut); IOUtils.closeStream(blockReceiver); IOUtils.closeStream(proxyReply); - IOUtils.closeStream(replyOut); } //update metrics diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto index 6283b569dda..9b4ba339d23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto @@ -207,7 +207,6 @@ enum Status { OOB_RESERVED1 = 9; // Reserved OOB_RESERVED2 = 10; // Reserved OOB_RESERVED3 = 11; // Reserved - IN_PROGRESS = 12; } message PipelineAckProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java index e0d79648a88..478b6d1546b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java @@ -272,10 +272,8 @@ public class TestBlockReplacement { // receiveResponse DataInputStream reply = new DataInputStream(sock.getInputStream()); - BlockOpResponseProto proto = BlockOpResponseProto.parseDelimitedFrom(reply); - while (proto.getStatus() == Status.IN_PROGRESS) { - proto = BlockOpResponseProto.parseDelimitedFrom(reply); - } + BlockOpResponseProto proto = + BlockOpResponseProto.parseDelimitedFrom(reply); return proto.getStatus() == Status.SUCCESS; } From 195961a7c1da86421761162836766b1de07930fd Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Wed, 13 Aug 2014 18:43:05 +0000 Subject: [PATCH 16/67] HDFS-6247. Avoid timeouts for replaceBlock() call by sending intermediate responses to Balancer (vinayakumarb) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617799 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/balancer/Dispatcher.java | 17 ++++------ .../hdfs/server/datanode/BlockReceiver.java | 32 ++++++++++++++++++- .../hdfs/server/datanode/DataXceiver.java | 9 +++--- .../src/main/proto/datatransfer.proto | 1 + .../server/datanode/TestBlockReplacement.java | 6 ++-- 6 files changed, 50 insertions(+), 18 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 685cd6211d6..9253b8b88a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -500,6 +500,9 @@ Release 2.6.0 - UNRELEASED HDFS-6830. BlockInfo.addStorage fails when DN changes the storage for a block replica (Arpit Agarwal) + HDFS-6247. Avoid timeouts for replaceBlock() call by sending intermediate + responses to Balancer (vinayakumarb) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index 4a6f96be7e5..b7dceb6f48d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -87,8 +87,6 @@ public class Dispatcher { private static final int MAX_NO_PENDING_MOVE_ITERATIONS = 5; private static final long DELAY_AFTER_ERROR = 10 * 1000L; // 10 seconds - private static final int BLOCK_MOVE_READ_TIMEOUT = 20 * 60 * 1000; // 20 - // minutes private final NameNodeConnector nnc; private final SaslDataTransferClient saslClient; @@ -278,13 +276,6 @@ public class Dispatcher { sock.connect( NetUtils.createSocketAddr(target.getDatanodeInfo().getXferAddr()), HdfsServerConstants.READ_TIMEOUT); - /* - * Unfortunately we don't have a good way to know if the Datanode is - * taking a really long time to move a block, OR something has gone - * wrong and it's never going to finish. To deal with this scenario, we - * set a long timeout (20 minutes) to avoid hanging indefinitely. - */ - sock.setSoTimeout(BLOCK_MOVE_READ_TIMEOUT); sock.setKeepAlive(true); @@ -341,8 +332,12 @@ public class Dispatcher { /** Receive a block copy response from the input stream */ private void receiveResponse(DataInputStream in) throws IOException { - BlockOpResponseProto response = BlockOpResponseProto - .parseFrom(vintPrefixed(in)); + BlockOpResponseProto response = + BlockOpResponseProto.parseFrom(vintPrefixed(in)); + while (response.getStatus() == Status.IN_PROGRESS) { + // read intermediate responses + response = BlockOpResponseProto.parseFrom(vintPrefixed(in)); + } if (response.getStatus() != Status.SUCCESS) { if (response.getStatus() == Status.ERROR_ACCESS_TOKEN) { throw new IOException("block move failed due to access token error"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index fb7ecd69e9a..9dcd006ed51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver; import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck; +import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; @@ -123,6 +124,14 @@ class BlockReceiver implements Closeable { private boolean syncOnClose; private long restartBudget; + /** + * for replaceBlock response + */ + private final long responseInterval; + private long lastResponseTime = 0; + private boolean isReplaceBlock = false; + private DataOutputStream replyOut = null; + BlockReceiver(final ExtendedBlock block, final StorageType storageType, final DataInputStream in, final String inAddr, final String myAddr, @@ -144,6 +153,9 @@ class BlockReceiver implements Closeable { this.isClient = !this.isDatanode; this.restartBudget = datanode.getDnConf().restartReplicaExpiry; this.datanodeSlowLogThresholdMs = datanode.getDnConf().datanodeSlowIoWarningThresholdMs; + // For replaceBlock() calls response should be sent to avoid socketTimeout + // at clients. So sending with the interval of 0.5 * socketTimeout + this.responseInterval = (long) (datanode.getDnConf().socketTimeout * 0.5); //for datanode, we have //1: clientName.length() == 0, and //2: stage == null or PIPELINE_SETUP_CREATE @@ -651,6 +663,20 @@ class BlockReceiver implements Closeable { lastPacketInBlock, offsetInBlock, Status.SUCCESS); } + /* + * Send in-progress responses for the replaceBlock() calls back to caller to + * avoid timeouts due to balancer throttling. HDFS-6247 + */ + if (isReplaceBlock + && (Time.monotonicNow() - lastResponseTime > responseInterval)) { + BlockOpResponseProto.Builder response = BlockOpResponseProto.newBuilder() + .setStatus(Status.IN_PROGRESS); + response.build().writeDelimitedTo(replyOut); + replyOut.flush(); + + lastResponseTime = Time.monotonicNow(); + } + if (throttler != null) { // throttle I/O throttler.throttle(len); } @@ -718,7 +744,8 @@ class BlockReceiver implements Closeable { DataInputStream mirrIn, // input from next datanode DataOutputStream replyOut, // output to previous datanode String mirrAddr, DataTransferThrottler throttlerArg, - DatanodeInfo[] downstreams) throws IOException { + DatanodeInfo[] downstreams, + boolean isReplaceBlock) throws IOException { syncOnClose = datanode.getDnConf().syncOnClose; boolean responderClosed = false; @@ -726,6 +753,9 @@ class BlockReceiver implements Closeable { mirrorAddr = mirrAddr; throttler = throttlerArg; + this.replyOut = replyOut; + this.isReplaceBlock = isReplaceBlock; + try { if (isClient && !isTransfer) { responder = new Daemon(datanode.threadGroup, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 5ef6cc7ee22..01fe036f1d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -708,7 +708,7 @@ class DataXceiver extends Receiver implements Runnable { if (blockReceiver != null) { String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, - mirrorAddr, null, targets); + mirrorAddr, null, targets, false); // send close-ack for transfer-RBW/Finalized if (isTransfer) { @@ -983,7 +983,7 @@ class DataXceiver extends Receiver implements Runnable { String errMsg = null; BlockReceiver blockReceiver = null; DataInputStream proxyReply = null; - + DataOutputStream replyOut = new DataOutputStream(getOutputStream()); try { // get the output stream to the proxy final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname); @@ -1040,8 +1040,8 @@ class DataXceiver extends Receiver implements Runnable { CachingStrategy.newDropBehind()); // receive a block - blockReceiver.receiveBlock(null, null, null, null, - dataXceiverServer.balanceThrottler, null); + blockReceiver.receiveBlock(null, null, replyOut, null, + dataXceiverServer.balanceThrottler, null, true); // notify name node datanode.notifyNamenodeReceivedBlock( @@ -1076,6 +1076,7 @@ class DataXceiver extends Receiver implements Runnable { IOUtils.closeStream(proxyOut); IOUtils.closeStream(blockReceiver); IOUtils.closeStream(proxyReply); + IOUtils.closeStream(replyOut); } //update metrics diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto index 9b4ba339d23..6283b569dda 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto @@ -207,6 +207,7 @@ enum Status { OOB_RESERVED1 = 9; // Reserved OOB_RESERVED2 = 10; // Reserved OOB_RESERVED3 = 11; // Reserved + IN_PROGRESS = 12; } message PipelineAckProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java index 478b6d1546b..e0d79648a88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java @@ -272,8 +272,10 @@ public class TestBlockReplacement { // receiveResponse DataInputStream reply = new DataInputStream(sock.getInputStream()); - BlockOpResponseProto proto = - BlockOpResponseProto.parseDelimitedFrom(reply); + BlockOpResponseProto proto = BlockOpResponseProto.parseDelimitedFrom(reply); + while (proto.getStatus() == Status.IN_PROGRESS) { + proto = BlockOpResponseProto.parseDelimitedFrom(reply); + } return proto.getStatus() == Status.SUCCESS; } From 6ab25dfcd9cedecd43c30039ace33d1c234a7b43 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 13 Aug 2014 19:10:24 +0000 Subject: [PATCH 17/67] HADOOP-8944. Shell command fs -count should include human readable option (Jonathan Allen via aw) (continued) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617802 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/fs/TestContentSummary.java | 248 ++++++++++++++++ .../org/apache/hadoop/fs/shell/TestCount.java | 270 ++++++++++++++++++ 2 files changed, 518 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java new file mode 100644 index 00000000000..9c8a8a4e067 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.junit.Test; +import org.mockito.InOrder; + +public class TestContentSummary { + + // check the empty constructor correctly initialises the object + @Test + public void testConstructorEmpty() { + ContentSummary contentSummary = new ContentSummary(); + assertEquals("getLength", 0, contentSummary.getLength()); + assertEquals("getFileCount", 0, contentSummary.getFileCount()); + assertEquals("getDirectoryCount", 0, contentSummary.getDirectoryCount()); + assertEquals("getQuota", 0, contentSummary.getQuota()); + assertEquals("getSpaceConsumed", 0, contentSummary.getSpaceConsumed()); + assertEquals("getSpaceQuota", 0, contentSummary.getSpaceQuota()); + } + + // check the full constructor with quota information + @Test + public void testConstructorWithQuota() { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + long quota = 44444; + long spaceConsumed = 55555; + long spaceQuota = 66666; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount, quota, spaceConsumed, spaceQuota); + assertEquals("getLength", length, contentSummary.getLength()); + assertEquals("getFileCount", fileCount, contentSummary.getFileCount()); + assertEquals("getDirectoryCount", directoryCount, + contentSummary.getDirectoryCount()); + assertEquals("getQuota", quota, contentSummary.getQuota()); + assertEquals("getSpaceConsumed", spaceConsumed, + contentSummary.getSpaceConsumed()); + assertEquals("getSpaceQuota", spaceQuota, contentSummary.getSpaceQuota()); + } + + // check the constructor with quota information + @Test + public void testConstructorNoQuota() { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount); + assertEquals("getLength", length, contentSummary.getLength()); + assertEquals("getFileCount", fileCount, contentSummary.getFileCount()); + assertEquals("getDirectoryCount", directoryCount, + contentSummary.getDirectoryCount()); + assertEquals("getQuota", -1, contentSummary.getQuota()); + assertEquals("getSpaceConsumed", length, contentSummary.getSpaceConsumed()); + assertEquals("getSpaceQuota", -1, contentSummary.getSpaceQuota()); + } + + // check the write method + @Test + public void testWrite() throws IOException { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + long quota = 44444; + long spaceConsumed = 55555; + long spaceQuota = 66666; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount, quota, spaceConsumed, spaceQuota); + + DataOutput out = mock(DataOutput.class); + InOrder inOrder = inOrder(out); + + contentSummary.write(out); + inOrder.verify(out).writeLong(length); + inOrder.verify(out).writeLong(fileCount); + inOrder.verify(out).writeLong(directoryCount); + inOrder.verify(out).writeLong(quota); + inOrder.verify(out).writeLong(spaceConsumed); + inOrder.verify(out).writeLong(spaceQuota); + } + + // check the readFields method + @Test + public void testReadFields() throws IOException { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + long quota = 44444; + long spaceConsumed = 55555; + long spaceQuota = 66666; + + ContentSummary contentSummary = new ContentSummary(); + + DataInput in = mock(DataInput.class); + when(in.readLong()).thenReturn(length).thenReturn(fileCount) + .thenReturn(directoryCount).thenReturn(quota).thenReturn(spaceConsumed) + .thenReturn(spaceQuota); + + contentSummary.readFields(in); + assertEquals("getLength", length, contentSummary.getLength()); + assertEquals("getFileCount", fileCount, contentSummary.getFileCount()); + assertEquals("getDirectoryCount", directoryCount, + contentSummary.getDirectoryCount()); + assertEquals("getQuota", quota, contentSummary.getQuota()); + assertEquals("getSpaceConsumed", spaceConsumed, + contentSummary.getSpaceConsumed()); + assertEquals("getSpaceQuota", spaceQuota, contentSummary.getSpaceQuota()); + } + + // check the header with quotas + @Test + public void testGetHeaderWithQuota() { + String header = " name quota rem name quota space quota " + + "rem space quota directories files bytes "; + assertEquals(header, ContentSummary.getHeader(true)); + } + + // check the header without quotas + @Test + public void testGetHeaderNoQuota() { + String header = " directories files bytes "; + assertEquals(header, ContentSummary.getHeader(false)); + } + + // check the toString method with quotas + @Test + public void testToStringWithQuota() { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + long quota = 44444; + long spaceConsumed = 55555; + long spaceQuota = 66665; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount, quota, spaceConsumed, spaceQuota); + String expected = " 44444 -11111 66665 11110" + + " 33333 22222 11111 "; + assertEquals(expected, contentSummary.toString(true)); + } + + // check the toString method with quotas + @Test + public void testToStringNoQuota() { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount); + String expected = " none inf none" + + " inf 33333 22222 11111 "; + assertEquals(expected, contentSummary.toString(true)); + } + + // check the toString method with quotas + @Test + public void testToStringNoShowQuota() { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + long quota = 44444; + long spaceConsumed = 55555; + long spaceQuota = 66665; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount, quota, spaceConsumed, spaceQuota); + String expected = " 33333 22222 11111 "; + assertEquals(expected, contentSummary.toString(false)); + } + + // check the toString method (defaults to with quotas) + @Test + public void testToString() { + long length = 11111; + long fileCount = 22222; + long directoryCount = 33333; + long quota = 44444; + long spaceConsumed = 55555; + long spaceQuota = 66665; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount, quota, spaceConsumed, spaceQuota); + String expected = " 44444 -11111 66665" + + " 11110 33333 22222 11111 "; + assertEquals(expected, contentSummary.toString()); + } + + // check the toString method with quotas + @Test + public void testToStringHumanWithQuota() { + long length = Long.MAX_VALUE; + long fileCount = 222222222; + long directoryCount = 33333; + long quota = 222256578; + long spaceConsumed = 1073741825; + long spaceQuota = 1; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount, quota, spaceConsumed, spaceQuota); + String expected = " 212.0 M 1023 1 " + + " -1 G 32.6 K 211.9 M 8.0 E "; + assertEquals(expected, contentSummary.toString(true, true)); + } + + // check the toString method with quotas + @Test + public void testToStringHumanNoShowQuota() { + long length = Long.MAX_VALUE; + long fileCount = 222222222; + long directoryCount = 33333; + long quota = 222256578; + long spaceConsumed = 55555; + long spaceQuota = Long.MAX_VALUE; + + ContentSummary contentSummary = new ContentSummary(length, fileCount, + directoryCount, quota, spaceConsumed, spaceQuota); + String expected = " 32.6 K 211.9 M 8.0 E "; + assertEquals(expected, contentSummary.toString(false, true)); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java new file mode 100644 index 00000000000..6c753c6a8f4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -0,0 +1,270 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.shell; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.io.PrintStream; +import java.io.IOException; +import java.net.URI; +import java.util.LinkedList; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Test; +import org.junit.Before; +import org.junit.BeforeClass; + +/** + * JUnit test class for {@link org.apache.hadoop.fs.shell.Count} + * + */ +public class TestCount { + private static final String WITH_QUOTAS = "Content summary with quotas"; + private static final String NO_QUOTAS = "Content summary without quotas"; + private static final String HUMAN = "human: "; + private static final String BYTES = "bytes: "; + private static Configuration conf; + private static FileSystem mockFs; + private static FileStatus fileStat; + private static ContentSummary mockCs; + + @BeforeClass + public static void setup() { + conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + mockFs = mock(FileSystem.class); + fileStat = mock(FileStatus.class); + mockCs = mock(ContentSummary.class); + when(fileStat.isFile()).thenReturn(true); + } + + @Before + public void resetMock() { + reset(mockFs); + } + + @Test + public void processOptionsHumanReadable() { + LinkedList options = new LinkedList(); + options.add("-h"); + options.add("dummy"); + Count count = new Count(); + count.processOptions(options); + assertFalse(count.isShowQuotas()); + assertTrue(count.isHumanReadable()); + } + + @Test + public void processOptionsAll() { + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("-h"); + options.add("dummy"); + Count count = new Count(); + count.processOptions(options); + assertTrue(count.isShowQuotas()); + assertTrue(count.isHumanReadable()); + } + + // check quotas are reported correctly + @Test + public void processPathShowQuotas() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + PathData pathData = new PathData(path.toString(), conf); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("dummy"); + count.processOptions(options); + + count.processPath(pathData); + verify(out).println(BYTES + WITH_QUOTAS + path.toString()); + verifyNoMoreInteractions(out); + } + + // check counts without quotas are reported correctly + @Test + public void processPathNoQuotas() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + PathData pathData = new PathData(path.toString(), conf); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("dummy"); + count.processOptions(options); + + count.processPath(pathData); + verify(out).println(BYTES + NO_QUOTAS + path.toString()); + verifyNoMoreInteractions(out); + } + + @Test + public void processPathShowQuotasHuman() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + PathData pathData = new PathData(path.toString(), conf); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("-q"); + options.add("-h"); + options.add("dummy"); + count.processOptions(options); + + count.processPath(pathData); + verify(out).println(HUMAN + WITH_QUOTAS + path.toString()); + } + + @Test + public void processPathNoQuotasHuman() throws Exception { + Path path = new Path("mockfs:/test"); + + when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); + PathData pathData = new PathData(path.toString(), conf); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + LinkedList options = new LinkedList(); + options.add("-h"); + options.add("dummy"); + count.processOptions(options); + + count.processPath(pathData); + verify(out).println(HUMAN + NO_QUOTAS + path.toString()); + } + + @Test + public void getCommandName() { + Count count = new Count(); + String actual = count.getCommandName(); + String expected = "count"; + assertEquals("Count.getCommandName", expected, actual); + } + + @Test + public void isDeprecated() { + Count count = new Count(); + boolean actual = count.isDeprecated(); + boolean expected = false; + assertEquals("Count.isDeprecated", expected, actual); + } + + @Test + public void getReplacementCommand() { + Count count = new Count(); + String actual = count.getReplacementCommand(); + String expected = null; + assertEquals("Count.getReplacementCommand", expected, actual); + } + + @Test + public void getName() { + Count count = new Count(); + String actual = count.getName(); + String expected = "count"; + assertEquals("Count.getName", expected, actual); + } + + @Test + public void getUsage() { + Count count = new Count(); + String actual = count.getUsage(); + String expected = "-count [-q] [-h] ..."; + assertEquals("Count.getUsage", expected, actual); + } + + + // mock content system + static class MockContentSummary extends ContentSummary { + + public MockContentSummary() {} + + @Override + public String toString(boolean qOption, boolean hOption) { + if (qOption) { + if (hOption) { + return(HUMAN + WITH_QUOTAS); + } else { + return(BYTES + WITH_QUOTAS); + } + } else { + if (hOption) { + return(HUMAN + NO_QUOTAS); + } else { + return(BYTES + NO_QUOTAS); + } + } + } + } + + // mock file system for use in testing + static class MockFileSystem extends FilterFileSystem { + Configuration conf; + + MockFileSystem() { + super(mockFs); + } + + @Override + public void initialize(URI uri, Configuration conf) { + this.conf = conf; + } + + @Override + public Path makeQualified(Path path) { + return path; + } + + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + return new MockContentSummary(); + } + + @Override + public Configuration getConf() { + return conf; + } + } +} From cc93c1fd759cecda57cd558b670c5fbcbca9b2e0 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 13 Aug 2014 19:18:50 +0000 Subject: [PATCH 18/67] MAPREDUCE-5944. Remove MRv1 commands from CommandsManual.apt.vm (Akira AJISAKA via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617804 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/site/apt/CommandsManual.apt.vm | 35 ------------------- hadoop-mapreduce-project/CHANGES.txt | 3 ++ 2 files changed, 3 insertions(+), 35 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm index 5ca5c2c5eb2..1e76bafec59 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm @@ -314,35 +314,6 @@ Administration Commands Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#dfsadmin} <<>>}} instead. -* <<>> - - Runs MR admin client - - Usage: <<>> - -*-------------------+-----------------------------------------------------------+ -|| COMMAND_OPTION || Description -*-------------------+-----------------------------------------------------------+ -| -refreshQueueAcls | Refresh the queue acls used by hadoop, to check access - | during submissions and administration of the job by the - | user. The properties present in mapred-queue-acls.xml is - | reloaded by the queue manager. -*-------------------+-----------------------------------------------------------+ - -* <<>> - - Runs the MapReduce job Tracker node. - - Usage: <<>> - -*--------------------+-----------------------------------------------------------+ -|| COMMAND_OPTION || Description -*--------------------+-----------------------------------------------------------+ -| -dumpConfiguration | Dumps the configuration used by the JobTracker alongwith - | queue configuration in JSON format into Standard output - | used by the jobtracker and exits. -*--------------------+-----------------------------------------------------------+ - * <<>> Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#namenode} @@ -352,9 +323,3 @@ Administration Commands Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#secondarynamenode} <<>>}} instead. - -* <<>> - - Runs a MapReduce task Tracker node. - - Usage: <<>> diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5bb579745a0..a683526ae60 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -151,6 +151,9 @@ Trunk (Unreleased) MAPREDUCE-5867. Fix NPE in KillAMPreemptionPolicy related to ProportionalCapacityPreemptionPolicy (Sunil G via devaraj) + MAPREDUCE-5944. Remove MRv1 commands from CommandsManual.apt.vm + (Akira AJISAKA via aw) + Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES From e51a5d691b004d49f7426d943568170126323929 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 13 Aug 2014 19:34:26 +0000 Subject: [PATCH 19/67] MAPREDUCE-5943. Separate mapred commands from CommandManual.apt.vm (Akira AJISAKA via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617806 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/site/apt/CommandsManual.apt.vm | 131 ++-------- hadoop-mapreduce-project/CHANGES.txt | 3 + .../src/site/apt/MapredCommands.apt.vm | 224 ++++++++++++++++++ hadoop-project/src/site/site.xml | 1 + 4 files changed, 242 insertions(+), 117 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm index 1e76bafec59..dd4eb0a3f6a 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm @@ -81,36 +81,15 @@ User Commands * <<>> - Creates a hadoop archive. More information can be found at Hadoop - Archives. - - Usage: <<* >>> - -*-------------------+-------------------------------------------------------+ -||COMMAND_OPTION || Description -*-------------------+-------------------------------------------------------+ -| -archiveName NAME | Name of the archive to be created. -*-------------------+-------------------------------------------------------+ -| src | Filesystem pathnames which work as usual with regular - | expressions. -*-------------------+-------------------------------------------------------+ -| dest | Destination directory which would contain the archive. -*-------------------+-------------------------------------------------------+ + Creates a hadoop archive. More information can be found at + {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopArchives.html} + Hadoop Archives Guide}}. * <<>> Copy file or directories recursively. More information can be found at - Hadoop DistCp Guide. - - Usage: << >>> - -*-------------------+--------------------------------------------+ -||COMMAND_OPTION || Description -*-------------------+--------------------------------------------+ -| srcurl | Source Url -*-------------------+--------------------------------------------+ -| desturl | Destination Url -*-------------------+--------------------------------------------+ + {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistCp.html} + Hadoop DistCp Guide}}. * <<>> @@ -142,103 +121,21 @@ User Commands * <<>> - Command to interact with Map Reduce Jobs. - - Usage: <<] | [-status ] | [-counter ] | [-kill ] | [-events <#-of-events>] | [-history [all] ] | [-list [all]] | [-kill-task ] | [-fail-task ] | [-set-priority ]>>> - -*------------------------------+---------------------------------------------+ -|| COMMAND_OPTION || Description -*------------------------------+---------------------------------------------+ -| -submit | Submits the job. -*------------------------------+---------------------------------------------+ -| -status | Prints the map and reduce completion - | percentage and all job counters. -*------------------------------+---------------------------------------------+ -| -counter | Prints the counter value. -*------------------------------+---------------------------------------------+ -| -kill | Kills the job. -*------------------------------+---------------------------------------------+ -| -events <#-of-events> | Prints the events' details - | received by jobtracker for the given range. -*------------------------------+---------------------------------------------+ -| -history [all] | Prints job details, failed and killed tip - | details. More details about the job such as - | successful tasks and task attempts made for - | each task can be viewed by specifying the [all] - | option. -*------------------------------+---------------------------------------------+ -| -list [all] | Displays jobs which are yet to complete. - | <<<-list all>>> displays all jobs. -*------------------------------+---------------------------------------------+ -| -kill-task | Kills the task. Killed tasks are NOT counted - | against failed attempts. -*------------------------------+---------------------------------------------+ -| -fail-task | Fails the task. Failed tasks are counted - | against failed attempts. -*------------------------------+---------------------------------------------+ -| -set-priority | Changes the priority of the job. Allowed - | priority values are VERY_HIGH, HIGH, NORMAL, - | LOW, VERY_LOW -*------------------------------+---------------------------------------------+ + Deprecated. Use + {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#job} + <<>>}} instead. * <<>> - Runs a pipes job. - - Usage: <<] [-jobconf , , - ...] [-input ] [-output ] [-jar ] [-inputformat - ] [-map ] [-partitioner ] [-reduce ] [-writer - ] [-program ] [-reduces ]>>> - -*----------------------------------------+------------------------------------+ -|| COMMAND_OPTION || Description -*----------------------------------------+------------------------------------+ -| -conf | Configuration for job -*----------------------------------------+------------------------------------+ -| -jobconf , , ... | Add/override configuration for job -*----------------------------------------+------------------------------------+ -| -input | Input directory -*----------------------------------------+------------------------------------+ -| -output | Output directory -*----------------------------------------+------------------------------------+ -| -jar | Jar filename -*----------------------------------------+------------------------------------+ -| -inputformat | InputFormat class -*----------------------------------------+------------------------------------+ -| -map | Java Map class -*----------------------------------------+------------------------------------+ -| -partitioner | Java Partitioner -*----------------------------------------+------------------------------------+ -| -reduce | Java Reduce class -*----------------------------------------+------------------------------------+ -| -writer | Java RecordWriter -*----------------------------------------+------------------------------------+ -| -program | Executable URI -*----------------------------------------+------------------------------------+ -| -reduces | Number of reduces -*----------------------------------------+------------------------------------+ + Deprecated. Use + {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#pipes} + <<>>}} instead. * <<>> - command to interact and view Job Queue information - - Usage: << [-showJobs]] | [-showacls]>>> - -*-----------------+-----------------------------------------------------------+ -|| COMMAND_OPTION || Description -*-----------------+-----------------------------------------------------------+ -| -list | Gets list of Job Queues configured in the system. - | Along with scheduling information associated with the job queues. -*-----------------+-----------------------------------------------------------+ -| -info [-showJobs] | Displays the job queue information and - | associated scheduling information of particular job queue. - | If <<<-showJobs>>> options is present a list of jobs - | submitted to the particular job queue is displayed. -*-----------------+-----------------------------------------------------------+ -| -showacls | Displays the queue name and associated queue operations - | allowed for the current user. The list consists of only - | those queues to which the user has access. -*-----------------+-----------------------------------------------------------+ + Deprecated. Use + {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#queue} + <<>>}} instead. * <<>> diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a683526ae60..22ca1b54e9b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -154,6 +154,9 @@ Trunk (Unreleased) MAPREDUCE-5944. Remove MRv1 commands from CommandsManual.apt.vm (Akira AJISAKA via aw) + MAPREDUCE-5943. Separate mapred commands from CommandManual.apt.vm + (Akira AJISAKA via aw) + Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm new file mode 100644 index 00000000000..795eb344e64 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm @@ -0,0 +1,224 @@ +~~ Licensed to the Apache Software Foundation (ASF) under one or more +~~ contributor license agreements. See the NOTICE file distributed with +~~ this work for additional information regarding copyright ownership. +~~ The ASF licenses this file to You under the Apache License, Version 2.0 +~~ (the "License"); you may not use this file except in compliance with +~~ the License. You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. + + --- + MapReduce Commands Guide + --- + --- + ${maven.build.timestamp} + +MapReduce Commands Guide + +%{toc|section=1|fromDepth=2|toDepth=4} + +* Overview + + MapReduce commands are invoked by the <<>> script. Running the + script without any arguments prints the description for all commands. + + Usage: <<>> + + MapReduce has an option parsing framework that employs parsing generic + options as well as running classes. + +*-------------------------+---------------------------------------------------+ +|| COMMAND_OPTIONS || Description | +*-------------------------+---------------------------------------------------+ +| --config confdir | Overwrites the default Configuration directory. Default +| | is $\{HADOOP_PREFIX\}/conf. +*-------------------------+---------------------------------------------------+ +| COMMAND COMMAND_OPTIONS | Various commands with their options are described +| | in the following sections. The commands have been +| | grouped into {{User Commands}} and +| | {{Administration Commands}}. +*-------------------------+---------------------------------------------------+ + +* User Commands + + Commands useful for users of a hadoop cluster. + +** <<>> + + Runs a pipes job. + + Usage: <<] [-jobconf , , + ...] [-input ] [-output ] [-jar ] [-inputformat + ] [-map ] [-partitioner ] [-reduce ] [-writer + ] [-program ] [-reduces ]>>> + +*----------------------------------------+------------------------------------+ +|| COMMAND_OPTION || Description +*----------------------------------------+------------------------------------+ +| -conf | Configuration for job +*----------------------------------------+------------------------------------+ +| -jobconf , , ... | Add/override configuration for job +*----------------------------------------+------------------------------------+ +| -input | Input directory +*----------------------------------------+------------------------------------+ +| -output | Output directory +*----------------------------------------+------------------------------------+ +| -jar | Jar filename +*----------------------------------------+------------------------------------+ +| -inputformat | InputFormat class +*----------------------------------------+------------------------------------+ +| -map | Java Map class +*----------------------------------------+------------------------------------+ +| -partitioner | Java Partitioner +*----------------------------------------+------------------------------------+ +| -reduce | Java Reduce class +*----------------------------------------+------------------------------------+ +| -writer | Java RecordWriter +*----------------------------------------+------------------------------------+ +| -program | Executable URI +*----------------------------------------+------------------------------------+ +| -reduces | Number of reduces +*----------------------------------------+------------------------------------+ + +** <<>> + + Command to interact with Map Reduce Jobs. + + Usage: <<] + | [-status ] + | [-counter ] + | [-kill ] + | [-events <#-of-events>] + | [-history [all] ] | [-list [all]] + | [-kill-task ] | [-fail-task ] + | [-set-priority ]>>> + +*------------------------------+---------------------------------------------+ +|| COMMAND_OPTION || Description +*------------------------------+---------------------------------------------+ +| -submit | Submits the job. +*------------------------------+---------------------------------------------+ +| -status | Prints the map and reduce completion + | percentage and all job counters. +*------------------------------+---------------------------------------------+ +| -counter | Prints the counter value. +*------------------------------+---------------------------------------------+ +| -kill | Kills the job. +*------------------------------+---------------------------------------------+ +| -events <#-of-events> | Prints the events' details + | received by jobtracker for the given range. +*------------------------------+---------------------------------------------+ +| -history [all] | Prints job details, failed and killed tip + | details. More details about the job such as + | successful tasks and task attempts made for + | each task can be viewed by specifying the + | [all] option. +*------------------------------+---------------------------------------------+ +| -list [all] | Displays jobs which are yet to complete. + | <<<-list all>>> displays all jobs. +*------------------------------+---------------------------------------------+ +| -kill-task | Kills the task. Killed tasks are NOT counted + | against failed attempts. +*------------------------------+---------------------------------------------+ +| -fail-task | Fails the task. Failed tasks are counted + | against failed attempts. +*------------------------------+---------------------------------------------+ +| -set-priority | Changes the priority of the job. Allowed + | priority values are VERY_HIGH, HIGH, NORMAL, + | LOW, VERY_LOW +*------------------------------+---------------------------------------------+ + +** <<>> + + command to interact and view Job Queue information + + Usage: << [-showJobs]] + | [-showacls]>>> + +*-----------------+-----------------------------------------------------------+ +|| COMMAND_OPTION || Description +*-----------------+-----------------------------------------------------------+ +| -list | Gets list of Job Queues configured in the system. + | Along with scheduling information associated with the job + | queues. +*-----------------+-----------------------------------------------------------+ +| -info [-showJobs] | Displays the job queue information and + | associated scheduling information of particular job queue. + | If <<<-showJobs>>> options is present a list of jobs + | submitted to the particular job queue is displayed. +*-----------------+-----------------------------------------------------------+ +| -showacls | Displays the queue name and associated queue operations + | allowed for the current user. The list consists of only + | those queues to which the user has access. +*-----------------+-----------------------------------------------------------+ + +** <<>> + + Prints the class path needed to get the Hadoop jar and the required + libraries. + + Usage: <<>> + +** <<>> + + Copy file or directories recursively. More information can be found at + {{{./DistCp.html}Hadoop DistCp Guide}}. + +** <<>> + + Creates a hadoop archive. More information can be found at + {{{./HadoopArchives.html}Hadoop Archives Guide}}. + +* Administration Commands + + Commands useful for administrators of a hadoop cluster. + +** <<>> + + Start JobHistoryServer. + + Usage: <<>> + +** <<>> + + Runs a MapReduce hsadmin client for execute JobHistoryServer administrative + commands. + + Usage: <<>> + +*-----------------+-----------------------------------------------------------+ +|| COMMAND_OPTION || Description +*-----------------+-----------------------------------------------------------+ +| -refreshUserToGroupsMappings | Refresh user-to-groups mappings +*-----------------+-----------------------------------------------------------+ +| -refreshSuperUserGroupsConfiguration| Refresh superuser proxy groups mappings +*-----------------+-----------------------------------------------------------+ +| -refreshAdminAcls | Refresh acls for administration of Job history server +*-----------------+-----------------------------------------------------------+ +| -refreshLoadedJobCache | Refresh loaded job cache of Job history server +*-----------------+-----------------------------------------------------------+ +| -refreshJobRetentionSettings|Refresh job history period, job cleaner settings +*-----------------+-----------------------------------------------------------+ +| -refreshLogRetentionSettings | Refresh log retention period and log retention +| | check interval +*-----------------+-----------------------------------------------------------+ +| -getGroups [username] | Get the groups which given user belongs to +*-----------------+-----------------------------------------------------------+ +| -help [cmd] | Displays help for the given command or all commands if none is +| | specified. +*-----------------+-----------------------------------------------------------+ diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 89f230fbc27..37e0153ec8f 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -95,6 +95,7 @@ + From 2f3ed504c1d5721185eb8244e7368df651ac72c1 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 13 Aug 2014 20:01:21 +0000 Subject: [PATCH 20/67] HADOOP-10966. Hadoop Common native compilation broken in windows. (Contributed by David Villegas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617823 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 547f2318feb..6a0fbc596e4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -575,6 +575,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10962. Flags for posix_fadvise are not valid in some architectures (David Villegas via Colin Patrick McCabe) + HADOOP-10966. Hadoop Common native compilation broken in windows. + (David Villegas via Arpit Agarwal) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index fa95047394f..d8538c8b0be 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -176,6 +176,7 @@ static void nioe_deinit(JNIEnv *env) { * If the value is not known, return the argument unchanged. */ static int map_fadvise_flag(jint flag) { +#ifdef HAVE_POSIX_FADVISE switch(flag) { case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_NORMAL: return POSIX_FADV_NORMAL; @@ -198,6 +199,9 @@ static int map_fadvise_flag(jint flag) { default: return flag; } +#else + return flag; +#endif } /* From 065d9ec5d93e37b4bbd6fd27cb411dc58b0fb686 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 13 Aug 2014 20:27:50 +0000 Subject: [PATCH 21/67] HDFS-6849. Replace HttpFS custom proxyuser handling with common implementation. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617831 13f79535-47bb-0310-9956-ffa450edef68 --- .../server/HttpFSAuthenticationFilter.java | 10 + .../http/server/HttpFSParametersProvider.java | 108 +++------ .../hadoop/fs/http/server/HttpFSServer.java | 144 ++++------- .../fs/http/server/HttpFSServerWebApp.java | 4 - .../apache/hadoop/lib/service/ProxyUser.java | 31 --- .../service/security/ProxyUserService.java | 179 -------------- .../apache/hadoop/lib/wsrs/UserProvider.java | 109 --------- .../src/main/resources/httpfs-default.xml | 6 +- .../security/TestProxyUserService.java | 226 ------------------ .../hadoop/lib/wsrs/TestUserProvider.java | 142 ----------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + 11 files changed, 93 insertions(+), 869 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/ProxyUser.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/ProxyUserService.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestProxyUserService.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java index d65616d45c6..8b332fc6e9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java @@ -91,4 +91,14 @@ public class HttpFSAuthenticationFilter return props; } + protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) { + Map proxyuserConf = HttpFSServerWebApp.get().getConfig(). + getValByRegex("httpfs\\.proxyuser\\."); + Configuration conf = new Configuration(false); + for (Map.Entry entry : proxyuserConf.entrySet()) { + conf.set(entry.getKey().substring("httpfs.".length()), entry.getValue()); + } + return conf; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index 84910d5de78..9b0be9bfc0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -30,8 +30,6 @@ import org.apache.hadoop.lib.wsrs.Param; import org.apache.hadoop.lib.wsrs.ParametersProvider; import org.apache.hadoop.lib.wsrs.ShortParam; import org.apache.hadoop.lib.wsrs.StringParam; -import org.apache.hadoop.lib.wsrs.UserProvider; -import org.slf4j.MDC; import javax.ws.rs.ext.Provider; import java.util.HashMap; @@ -53,57 +51,44 @@ public class HttpFSParametersProvider extends ParametersProvider { static { PARAMS_DEF.put(Operation.OPEN, - new Class[]{DoAsParam.class, OffsetParam.class, LenParam.class}); - PARAMS_DEF.put(Operation.GETFILESTATUS, new Class[]{DoAsParam.class}); - PARAMS_DEF.put(Operation.LISTSTATUS, - new Class[]{DoAsParam.class, FilterParam.class}); - PARAMS_DEF.put(Operation.GETHOMEDIRECTORY, new Class[]{DoAsParam.class}); - PARAMS_DEF.put(Operation.GETCONTENTSUMMARY, new Class[]{DoAsParam.class}); - PARAMS_DEF.put(Operation.GETFILECHECKSUM, new Class[]{DoAsParam.class}); - PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, - new Class[]{DoAsParam.class}); - PARAMS_DEF.put(Operation.GETACLSTATUS, new Class[]{DoAsParam.class}); - PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{DoAsParam.class}); - PARAMS_DEF.put(Operation.APPEND, - new Class[]{DoAsParam.class, DataParam.class}); + new Class[]{OffsetParam.class, LenParam.class}); + PARAMS_DEF.put(Operation.GETFILESTATUS, new Class[]{}); + PARAMS_DEF.put(Operation.LISTSTATUS, new Class[]{FilterParam.class}); + PARAMS_DEF.put(Operation.GETHOMEDIRECTORY, new Class[]{}); + PARAMS_DEF.put(Operation.GETCONTENTSUMMARY, new Class[]{}); + PARAMS_DEF.put(Operation.GETFILECHECKSUM, new Class[]{}); + PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, new Class[]{}); + PARAMS_DEF.put(Operation.GETACLSTATUS, new Class[]{}); + PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{}); + PARAMS_DEF.put(Operation.APPEND, new Class[]{DataParam.class}); PARAMS_DEF.put(Operation.CONCAT, new Class[]{SourcesParam.class}); PARAMS_DEF.put(Operation.CREATE, - new Class[]{DoAsParam.class, PermissionParam.class, OverwriteParam.class, + new Class[]{PermissionParam.class, OverwriteParam.class, ReplicationParam.class, BlockSizeParam.class, DataParam.class}); - PARAMS_DEF.put(Operation.MKDIRS, - new Class[]{DoAsParam.class, PermissionParam.class}); - PARAMS_DEF.put(Operation.RENAME, - new Class[]{DoAsParam.class, DestinationParam.class}); + PARAMS_DEF.put(Operation.MKDIRS, new Class[]{PermissionParam.class}); + PARAMS_DEF.put(Operation.RENAME, new Class[]{DestinationParam.class}); PARAMS_DEF.put(Operation.SETOWNER, - new Class[]{DoAsParam.class, OwnerParam.class, GroupParam.class}); - PARAMS_DEF.put(Operation.SETPERMISSION, - new Class[]{DoAsParam.class, PermissionParam.class}); + new Class[]{OwnerParam.class, GroupParam.class}); + PARAMS_DEF.put(Operation.SETPERMISSION, new Class[]{PermissionParam.class}); PARAMS_DEF.put(Operation.SETREPLICATION, - new Class[]{DoAsParam.class, ReplicationParam.class}); + new Class[]{ReplicationParam.class}); PARAMS_DEF.put(Operation.SETTIMES, - new Class[]{DoAsParam.class, ModifiedTimeParam.class, - AccessTimeParam.class}); - PARAMS_DEF.put(Operation.DELETE, - new Class[]{DoAsParam.class, RecursiveParam.class}); - PARAMS_DEF.put(Operation.SETACL, - new Class[]{DoAsParam.class, AclPermissionParam.class}); - PARAMS_DEF.put(Operation.REMOVEACL, - new Class[]{DoAsParam.class}); + new Class[]{ModifiedTimeParam.class, AccessTimeParam.class}); + PARAMS_DEF.put(Operation.DELETE, new Class[]{RecursiveParam.class}); + PARAMS_DEF.put(Operation.SETACL, new Class[]{AclPermissionParam.class}); + PARAMS_DEF.put(Operation.REMOVEACL, new Class[]{}); PARAMS_DEF.put(Operation.MODIFYACLENTRIES, - new Class[]{DoAsParam.class, AclPermissionParam.class}); + new Class[]{AclPermissionParam.class}); PARAMS_DEF.put(Operation.REMOVEACLENTRIES, - new Class[]{DoAsParam.class, AclPermissionParam.class}); - PARAMS_DEF.put(Operation.REMOVEDEFAULTACL, - new Class[]{DoAsParam.class}); + new Class[]{AclPermissionParam.class}); + PARAMS_DEF.put(Operation.REMOVEDEFAULTACL, new Class[]{}); PARAMS_DEF.put(Operation.SETXATTR, - new Class[]{DoAsParam.class, XAttrNameParam.class, XAttrValueParam.class, + new Class[]{XAttrNameParam.class, XAttrValueParam.class, XAttrSetFlagParam.class}); - PARAMS_DEF.put(Operation.REMOVEXATTR, - new Class[]{DoAsParam.class, XAttrNameParam.class}); + PARAMS_DEF.put(Operation.REMOVEXATTR, new Class[]{XAttrNameParam.class}); PARAMS_DEF.put(Operation.GETXATTRS, - new Class[]{DoAsParam.class, XAttrNameParam.class, XAttrEncodingParam.class}); - PARAMS_DEF.put(Operation.LISTXATTRS, - new Class[]{DoAsParam.class}); + new Class[]{XAttrNameParam.class, XAttrEncodingParam.class}); + PARAMS_DEF.put(Operation.LISTXATTRS, new Class[]{}); } public HttpFSParametersProvider() { @@ -205,41 +190,6 @@ public class HttpFSParametersProvider extends ParametersProvider { } } - /** - * Class for do-as parameter. - */ - @InterfaceAudience.Private - public static class DoAsParam extends StringParam { - - /** - * Parameter name. - */ - public static final String NAME = HttpFSFileSystem.DO_AS_PARAM; - - /** - * Constructor. - */ - public DoAsParam() { - super(NAME, null, UserProvider.getUserPattern()); - } - - /** - * Delegates to parent and then adds do-as user to - * MDC context for logging purposes. - * - * - * @param str parameter value. - * - * @return parsed parameter - */ - @Override - public String parseParam(String str) { - String doAs = super.parseParam(str); - MDC.put(getName(), (doAs != null) ? doAs : "-"); - return doAs; - } - } - /** * Class for filter parameter. */ @@ -275,7 +225,7 @@ public class HttpFSParametersProvider extends ParametersProvider { * Constructor. */ public GroupParam() { - super(NAME, null, UserProvider.getUserPattern()); + super(NAME, null); } } @@ -371,7 +321,7 @@ public class HttpFSParametersProvider extends ParametersProvider { * Constructor. */ public OwnerParam() { - super(NAME, null, UserProvider.getUserPattern()); + super(NAME, null); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index 000d34ec037..db113611ae3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -29,7 +29,6 @@ import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.AclPermissionPa import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.BlockSizeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DataParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam; -import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DoAsParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.FilterParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.GroupParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.LenParam; @@ -50,12 +49,11 @@ import org.apache.hadoop.lib.service.FileSystemAccess; import org.apache.hadoop.lib.service.FileSystemAccessException; import org.apache.hadoop.lib.service.Groups; import org.apache.hadoop.lib.service.Instrumentation; -import org.apache.hadoop.lib.service.ProxyUser; import org.apache.hadoop.lib.servlet.FileSystemReleaseFilter; -import org.apache.hadoop.lib.servlet.HostnameFilter; import org.apache.hadoop.lib.wsrs.InputStreamEntity; import org.apache.hadoop.lib.wsrs.Parameters; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,7 +77,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.security.AccessControlException; -import java.security.Principal; import java.text.MessageFormat; import java.util.EnumSet; import java.util.List; @@ -96,49 +93,11 @@ import java.util.Map; public class HttpFSServer { private static Logger AUDIT_LOG = LoggerFactory.getLogger("httpfsaudit"); - /** - * Resolves the effective user that will be used to request a FileSystemAccess filesystem. - *

- * If the doAs-user is NULL or the same as the user, it returns the user. - *

- * Otherwise it uses proxyuser rules (see {@link ProxyUser} to determine if the - * current user can impersonate the doAs-user. - *

- * If the current user cannot impersonate the doAs-user an - * AccessControlException will be thrown. - * - * @param user principal for whom the filesystem instance is. - * @param doAs do-as user, if any. - * - * @return the effective user. - * - * @throws IOException thrown if an IO error occurrs. - * @throws AccessControlException thrown if the current user cannot impersonate - * the doAs-user. - */ - private String getEffectiveUser(Principal user, String doAs) throws IOException { - String effectiveUser = user.getName(); - if (doAs != null && !doAs.equals(user.getName())) { - ProxyUser proxyUser = HttpFSServerWebApp.get().get(ProxyUser.class); - String proxyUserName; - if (user instanceof AuthenticationToken) { - proxyUserName = ((AuthenticationToken)user).getUserName(); - } else { - proxyUserName = user.getName(); - } - proxyUser.validate(proxyUserName, HostnameFilter.get(), doAs); - effectiveUser = doAs; - AUDIT_LOG.info("Proxy user [{}] DoAs user [{}]", proxyUserName, doAs); - } - return effectiveUser; - } - /** * Executes a {@link FileSystemAccess.FileSystemExecutor} using a filesystem for the effective * user. * - * @param user principal making the request. - * @param doAs do-as user, if any. + * @param ugi user making the request. * @param executor FileSystemExecutor to execute. * * @return FileSystemExecutor response @@ -147,12 +106,11 @@ public class HttpFSServer { * @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown * exceptions are handled by {@link HttpFSExceptionProvider}. */ - private T fsExecute(Principal user, String doAs, FileSystemAccess.FileSystemExecutor executor) + private T fsExecute(UserGroupInformation ugi, FileSystemAccess.FileSystemExecutor executor) throws IOException, FileSystemAccessException { - String hadoopUser = getEffectiveUser(user, doAs); FileSystemAccess fsAccess = HttpFSServerWebApp.get().get(FileSystemAccess.class); Configuration conf = HttpFSServerWebApp.get().get(FileSystemAccess.class).getFileSystemConfiguration(); - return fsAccess.execute(hadoopUser, conf, executor); + return fsAccess.execute(ugi.getShortUserName(), conf, executor); } /** @@ -162,8 +120,7 @@ public class HttpFSServer { * If a do-as user is specified, the current user must be a valid proxyuser, otherwise an * AccessControlException will be thrown. * - * @param user principal for whom the filesystem instance is. - * @param doAs do-as user, if any. + * @param ugi principal for whom the filesystem instance is. * * @return a filesystem for the specified user or do-as user. * @@ -172,8 +129,9 @@ public class HttpFSServer { * @throws FileSystemAccessException thrown if a FileSystemAccess releated error occurred. Thrown * exceptions are handled by {@link HttpFSExceptionProvider}. */ - private FileSystem createFileSystem(Principal user, String doAs) throws IOException, FileSystemAccessException { - String hadoopUser = getEffectiveUser(user, doAs); + private FileSystem createFileSystem(UserGroupInformation ugi) + throws IOException, FileSystemAccessException { + String hadoopUser = ugi.getShortUserName(); FileSystemAccess fsAccess = HttpFSServerWebApp.get().get(FileSystemAccess.class); Configuration conf = HttpFSServerWebApp.get().get(FileSystemAccess.class).getFileSystemConfiguration(); FileSystem fs = fsAccess.createFileSystem(hadoopUser, conf); @@ -192,7 +150,6 @@ public class HttpFSServer { /** * Special binding for '/' as it is not handled by the wildcard binding. * - * @param user the principal of the user making the request. * @param op the HttpFS operation of the request. * @param params the HttpFS parameters of the request. * @@ -206,11 +163,10 @@ public class HttpFSServer { */ @GET @Produces(MediaType.APPLICATION_JSON) - public Response getRoot(@Context Principal user, - @QueryParam(OperationParam.NAME) OperationParam op, + public Response getRoot(@QueryParam(OperationParam.NAME) OperationParam op, @Context Parameters params) throws IOException, FileSystemAccessException { - return get(user, "", op, params); + return get("", op, params); } private String makeAbsolute(String path) { @@ -220,7 +176,6 @@ public class HttpFSServer { /** * Binding to handle GET requests, supported operations are * - * @param user the principal of the user making the request. * @param path the path for operation. * @param op the HttpFS operation of the request. * @param params the HttpFS parameters of the request. @@ -236,21 +191,20 @@ public class HttpFSServer { @GET @Path("{path:.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) - public Response get(@Context Principal user, - @PathParam("path") String path, + public Response get(@PathParam("path") String path, @QueryParam(OperationParam.NAME) OperationParam op, @Context Parameters params) throws IOException, FileSystemAccessException { + UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); - String doAs = params.get(DoAsParam.NAME, DoAsParam.class); switch (op.value()) { case OPEN: { //Invoking the command directly using an unmanaged FileSystem that is // released by the FileSystemReleaseFilter FSOperations.FSOpen command = new FSOperations.FSOpen(path); - FileSystem fs = createFileSystem(user, doAs); + FileSystem fs = createFileSystem(user); InputStream is = command.execute(fs); Long offset = params.get(OffsetParam.NAME, OffsetParam.class); Long len = params.get(LenParam.NAME, LenParam.class); @@ -264,7 +218,7 @@ public class HttpFSServer { case GETFILESTATUS: { FSOperations.FSFileStatus command = new FSOperations.FSFileStatus(path); - Map json = fsExecute(user, doAs, command); + Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -273,7 +227,7 @@ public class HttpFSServer { String filter = params.get(FilterParam.NAME, FilterParam.class); FSOperations.FSListStatus command = new FSOperations.FSListStatus( path, filter); - Map json = fsExecute(user, doAs, command); + Map json = fsExecute(user, command); AUDIT_LOG.info("[{}] filter [{}]", path, (filter != null) ? filter : "-"); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); @@ -282,7 +236,7 @@ public class HttpFSServer { case GETHOMEDIRECTORY: { enforceRootPath(op.value(), path); FSOperations.FSHomeDir command = new FSOperations.FSHomeDir(); - JSONObject json = fsExecute(user, doAs, command); + JSONObject json = fsExecute(user, command); AUDIT_LOG.info(""); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -290,7 +244,7 @@ public class HttpFSServer { case INSTRUMENTATION: { enforceRootPath(op.value(), path); Groups groups = HttpFSServerWebApp.get().get(Groups.class); - List userGroups = groups.getGroups(user.getName()); + List userGroups = groups.getGroups(user.getShortUserName()); if (!userGroups.contains(HttpFSServerWebApp.get().getAdminGroup())) { throw new AccessControlException( "User not in HttpFSServer admin group"); @@ -304,7 +258,7 @@ public class HttpFSServer { case GETCONTENTSUMMARY: { FSOperations.FSContentSummary command = new FSOperations.FSContentSummary(path); - Map json = fsExecute(user, doAs, command); + Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -312,7 +266,7 @@ public class HttpFSServer { case GETFILECHECKSUM: { FSOperations.FSFileChecksum command = new FSOperations.FSFileChecksum(path); - Map json = fsExecute(user, doAs, command); + Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -324,7 +278,7 @@ public class HttpFSServer { case GETACLSTATUS: { FSOperations.FSAclStatus command = new FSOperations.FSAclStatus(path); - Map json = fsExecute(user, doAs, command); + Map json = fsExecute(user, command); AUDIT_LOG.info("ACL status for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -337,7 +291,7 @@ public class HttpFSServer { FSOperations.FSGetXAttrs command = new FSOperations.FSGetXAttrs(path, xattrNames, encoding); @SuppressWarnings("rawtypes") - Map json = fsExecute(user, doAs, command); + Map json = fsExecute(user, command); AUDIT_LOG.info("XAttrs for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -345,7 +299,7 @@ public class HttpFSServer { case LISTXATTRS: { FSOperations.FSListXAttrs command = new FSOperations.FSListXAttrs(path); @SuppressWarnings("rawtypes") - Map json = fsExecute(user, doAs, command); + Map json = fsExecute(user, command); AUDIT_LOG.info("XAttr names for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -363,7 +317,6 @@ public class HttpFSServer { /** * Binding to handle DELETE requests. * - * @param user the principal of the user making the request. * @param path the path for operation. * @param op the HttpFS operation of the request. * @param params the HttpFS parameters of the request. @@ -379,15 +332,14 @@ public class HttpFSServer { @DELETE @Path("{path:.*}") @Produces(MediaType.APPLICATION_JSON) - public Response delete(@Context Principal user, - @PathParam("path") String path, - @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params) + public Response delete(@PathParam("path") String path, + @QueryParam(OperationParam.NAME) OperationParam op, + @Context Parameters params) throws IOException, FileSystemAccessException { + UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); - String doAs = params.get(DoAsParam.NAME, DoAsParam.class); switch (op.value()) { case DELETE: { Boolean recursive = @@ -395,7 +347,7 @@ public class HttpFSServer { AUDIT_LOG.info("[{}] recursive [{}]", path, recursive); FSOperations.FSDelete command = new FSOperations.FSDelete(path, recursive); - JSONObject json = fsExecute(user, doAs, command); + JSONObject json = fsExecute(user, command); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } @@ -412,7 +364,6 @@ public class HttpFSServer { * Binding to handle POST requests. * * @param is the inputstream for the request payload. - * @param user the principal of the user making the request. * @param uriInfo the of the request. * @param path the path for operation. * @param op the HttpFS operation of the request. @@ -431,18 +382,17 @@ public class HttpFSServer { @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_JSON}) public Response post(InputStream is, - @Context Principal user, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam(OperationParam.NAME) OperationParam op, @Context Parameters params) throws IOException, FileSystemAccessException { + UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); switch (op.value()) { case APPEND: { - String doAs = params.get(DoAsParam.NAME, DoAsParam.class); Boolean hasData = params.get(DataParam.NAME, DataParam.class); if (!hasData) { response = Response.temporaryRedirect( @@ -451,7 +401,7 @@ public class HttpFSServer { } else { FSOperations.FSAppend command = new FSOperations.FSAppend(is, path); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok().type(MediaType.APPLICATION_JSON).build(); } @@ -463,7 +413,7 @@ public class HttpFSServer { FSOperations.FSConcat command = new FSOperations.FSConcat(path, sources.split(",")); - fsExecute(user, null, command); + fsExecute(user, command); AUDIT_LOG.info("[{}]", path); System.out.println("SENT RESPONSE"); response = Response.ok().build(); @@ -498,7 +448,6 @@ public class HttpFSServer { * Binding to handle PUT requests. * * @param is the inputstream for the request payload. - * @param user the principal of the user making the request. * @param uriInfo the of the request. * @param path the path for operation. * @param op the HttpFS operation of the request. @@ -517,16 +466,15 @@ public class HttpFSServer { @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_JSON}) public Response put(InputStream is, - @Context Principal user, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam(OperationParam.NAME) OperationParam op, @Context Parameters params) throws IOException, FileSystemAccessException { + UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); - String doAs = params.get(DoAsParam.NAME, DoAsParam.class); switch (op.value()) { case CREATE: { Boolean hasData = params.get(DataParam.NAME, DataParam.class); @@ -546,7 +494,7 @@ public class HttpFSServer { FSOperations.FSCreate command = new FSOperations.FSCreate(is, path, permission, override, replication, blockSize); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info( "[{}] permission [{}] override [{}] replication [{}] blockSize [{}]", new Object[]{path, permission, override, replication, blockSize}); @@ -564,7 +512,7 @@ public class HttpFSServer { FSOperations.FSSetXAttr command = new FSOperations.FSSetXAttr( path, xattrName, xattrValue, flag); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] to xAttr [{}]", path, xattrName); response = Response.ok().build(); break; @@ -573,7 +521,7 @@ public class HttpFSServer { String xattrName = params.get(XAttrNameParam.NAME, XAttrNameParam.class); FSOperations.FSRemoveXAttr command = new FSOperations.FSRemoveXAttr( path, xattrName); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] removed xAttr [{}]", path, xattrName); response = Response.ok().build(); break; @@ -583,7 +531,7 @@ public class HttpFSServer { PermissionParam.class); FSOperations.FSMkdirs command = new FSOperations.FSMkdirs(path, permission); - JSONObject json = fsExecute(user, doAs, command); + JSONObject json = fsExecute(user, command); AUDIT_LOG.info("[{}] permission [{}]", path, permission); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -592,7 +540,7 @@ public class HttpFSServer { String toPath = params.get(DestinationParam.NAME, DestinationParam.class); FSOperations.FSRename command = new FSOperations.FSRename(path, toPath); - JSONObject json = fsExecute(user, doAs, command); + JSONObject json = fsExecute(user, command); AUDIT_LOG.info("[{}] to [{}]", path, toPath); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; @@ -602,7 +550,7 @@ public class HttpFSServer { String group = params.get(GroupParam.NAME, GroupParam.class); FSOperations.FSSetOwner command = new FSOperations.FSSetOwner(path, owner, group); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] to (O/G)[{}]", path, owner + ":" + group); response = Response.ok().build(); break; @@ -612,7 +560,7 @@ public class HttpFSServer { PermissionParam.class); FSOperations.FSSetPermission command = new FSOperations.FSSetPermission(path, permission); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] to [{}]", path, permission); response = Response.ok().build(); break; @@ -622,7 +570,7 @@ public class HttpFSServer { ReplicationParam.class); FSOperations.FSSetReplication command = new FSOperations.FSSetReplication(path, replication); - JSONObject json = fsExecute(user, doAs, command); + JSONObject json = fsExecute(user, command); AUDIT_LOG.info("[{}] to [{}]", path, replication); response = Response.ok(json).build(); break; @@ -634,7 +582,7 @@ public class HttpFSServer { AccessTimeParam.class); FSOperations.FSSetTimes command = new FSOperations.FSSetTimes(path, modifiedTime, accessTime); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] to (M/A)[{}]", path, modifiedTime + ":" + accessTime); response = Response.ok().build(); @@ -645,7 +593,7 @@ public class HttpFSServer { AclPermissionParam.class); FSOperations.FSSetAcl command = new FSOperations.FSSetAcl(path, aclSpec); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] to acl [{}]", path, aclSpec); response = Response.ok().build(); break; @@ -653,7 +601,7 @@ public class HttpFSServer { case REMOVEACL: { FSOperations.FSRemoveAcl command = new FSOperations.FSRemoveAcl(path); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] removed acl", path); response = Response.ok().build(); break; @@ -663,7 +611,7 @@ public class HttpFSServer { AclPermissionParam.class); FSOperations.FSModifyAclEntries command = new FSOperations.FSModifyAclEntries(path, aclSpec); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] modify acl entry with [{}]", path, aclSpec); response = Response.ok().build(); break; @@ -673,7 +621,7 @@ public class HttpFSServer { AclPermissionParam.class); FSOperations.FSRemoveAclEntries command = new FSOperations.FSRemoveAclEntries(path, aclSpec); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] remove acl entry [{}]", path, aclSpec); response = Response.ok().build(); break; @@ -681,7 +629,7 @@ public class HttpFSServer { case REMOVEDEFAULTACL: { FSOperations.FSRemoveDefaultAcl command = new FSOperations.FSRemoveDefaultAcl(path); - fsExecute(user, doAs, command); + fsExecute(user, command); AUDIT_LOG.info("[{}] remove default acl", path); response = Response.ok().build(); break; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java index 258a5c1e3ae..b7ae3015e3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.lib.server.ServerException; import org.apache.hadoop.lib.service.FileSystemAccess; import org.apache.hadoop.lib.servlet.ServerWebApp; -import org.apache.hadoop.lib.wsrs.UserProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,9 +102,6 @@ public class HttpFSServerWebApp extends ServerWebApp { LOG.info("Connects to Namenode [{}]", get().get(FileSystemAccess.class).getFileSystemConfiguration(). get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)); - String userPattern = getConfig().get(UserProvider.USER_PATTERN_KEY, - UserProvider.USER_PATTERN_DEFAULT); - UserProvider.setUserPattern(userPattern); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/ProxyUser.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/ProxyUser.java deleted file mode 100644 index 7619dcd200c..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/ProxyUser.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.service; - -import org.apache.hadoop.classification.InterfaceAudience; - -import java.io.IOException; -import java.security.AccessControlException; - -@InterfaceAudience.Private -public interface ProxyUser { - - public void validate(String proxyUser, String proxyHost, String doAsUser) throws IOException, AccessControlException; - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/ProxyUserService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/ProxyUserService.java deleted file mode 100644 index 095ce11b4dc..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/ProxyUserService.java +++ /dev/null @@ -1,179 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.service.security; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.lib.lang.XException; -import org.apache.hadoop.lib.server.BaseService; -import org.apache.hadoop.lib.server.ServiceException; -import org.apache.hadoop.lib.service.Groups; -import org.apache.hadoop.lib.service.ProxyUser; -import org.apache.hadoop.lib.util.Check; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetAddress; -import java.security.AccessControlException; -import java.text.MessageFormat; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@InterfaceAudience.Private -public class ProxyUserService extends BaseService implements ProxyUser { - private static Logger LOG = LoggerFactory.getLogger(ProxyUserService.class); - - @InterfaceAudience.Private - public static enum ERROR implements XException.ERROR { - PRXU01("Could not normalize host name [{0}], {1}"), - PRXU02("Missing [{0}] property"); - - private String template; - - ERROR(String template) { - this.template = template; - } - - @Override - public String getTemplate() { - return template; - } - } - - private static final String PREFIX = "proxyuser"; - private static final String GROUPS = ".groups"; - private static final String HOSTS = ".hosts"; - - private Map> proxyUserHosts = new HashMap>(); - private Map> proxyUserGroups = new HashMap>(); - - public ProxyUserService() { - super(PREFIX); - } - - @Override - public Class getInterface() { - return ProxyUser.class; - } - - @Override - public Class[] getServiceDependencies() { - return new Class[]{Groups.class}; - } - - @Override - protected void init() throws ServiceException { - for (Map.Entry entry : getServiceConfig()) { - String key = entry.getKey(); - if (key.endsWith(GROUPS)) { - String proxyUser = key.substring(0, key.lastIndexOf(GROUPS)); - if (getServiceConfig().get(proxyUser + HOSTS) == null) { - throw new ServiceException(ERROR.PRXU02, getPrefixedName(proxyUser + HOSTS)); - } - String value = entry.getValue().trim(); - LOG.info("Loading proxyuser settings [{}]=[{}]", key, value); - Set values = null; - if (!value.equals("*")) { - values = new HashSet(Arrays.asList(value.split(","))); - } - proxyUserGroups.put(proxyUser, values); - } - if (key.endsWith(HOSTS)) { - String proxyUser = key.substring(0, key.lastIndexOf(HOSTS)); - if (getServiceConfig().get(proxyUser + GROUPS) == null) { - throw new ServiceException(ERROR.PRXU02, getPrefixedName(proxyUser + GROUPS)); - } - String value = entry.getValue().trim(); - LOG.info("Loading proxyuser settings [{}]=[{}]", key, value); - Set values = null; - if (!value.equals("*")) { - String[] hosts = value.split(","); - for (int i = 0; i < hosts.length; i++) { - String originalName = hosts[i]; - try { - hosts[i] = normalizeHostname(originalName); - } catch (Exception ex) { - throw new ServiceException(ERROR.PRXU01, originalName, ex.getMessage(), ex); - } - LOG.info(" Hostname, original [{}], normalized [{}]", originalName, hosts[i]); - } - values = new HashSet(Arrays.asList(hosts)); - } - proxyUserHosts.put(proxyUser, values); - } - } - } - - @Override - public void validate(String proxyUser, String proxyHost, String doAsUser) throws IOException, - AccessControlException { - Check.notEmpty(proxyUser, "proxyUser"); - Check.notEmpty(proxyHost, "proxyHost"); - Check.notEmpty(doAsUser, "doAsUser"); - LOG.debug("Authorization check proxyuser [{}] host [{}] doAs [{}]", - new Object[]{proxyUser, proxyHost, doAsUser}); - if (proxyUserHosts.containsKey(proxyUser)) { - proxyHost = normalizeHostname(proxyHost); - validateRequestorHost(proxyUser, proxyHost, proxyUserHosts.get(proxyUser)); - validateGroup(proxyUser, doAsUser, proxyUserGroups.get(proxyUser)); - } else { - throw new AccessControlException(MessageFormat.format("User [{0}] not defined as proxyuser", proxyUser)); - } - } - - private void validateRequestorHost(String proxyUser, String hostname, Set validHosts) - throws IOException, AccessControlException { - if (validHosts != null) { - if (!validHosts.contains(hostname) && !validHosts.contains(normalizeHostname(hostname))) { - throw new AccessControlException(MessageFormat.format("Unauthorized host [{0}] for proxyuser [{1}]", - hostname, proxyUser)); - } - } - } - - private void validateGroup(String proxyUser, String user, Set validGroups) throws IOException, - AccessControlException { - if (validGroups != null) { - List userGroups = getServer().get(Groups.class).getGroups(user); - for (String g : validGroups) { - if (userGroups.contains(g)) { - return; - } - } - throw new AccessControlException( - MessageFormat.format("Unauthorized proxyuser [{0}] for user [{1}], not in proxyuser groups", - proxyUser, user)); - } - } - - private String normalizeHostname(String name) { - try { - InetAddress address = InetAddress.getByName(name); - return address.getCanonicalHostName(); - } catch (IOException ex) { - throw new AccessControlException(MessageFormat.format("Could not resolve host [{0}], {1}", name, - ex.getMessage())); - } - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java deleted file mode 100644 index cd1cfc7f4ef..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/UserProvider.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - -import com.sun.jersey.api.core.HttpContext; -import com.sun.jersey.core.spi.component.ComponentContext; -import com.sun.jersey.core.spi.component.ComponentScope; -import com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable; -import com.sun.jersey.spi.inject.Injectable; -import com.sun.jersey.spi.inject.InjectableProvider; -import org.apache.hadoop.classification.InterfaceAudience; -import org.slf4j.MDC; - -import javax.ws.rs.core.Context; -import javax.ws.rs.ext.Provider; -import java.lang.reflect.Type; -import java.security.Principal; -import java.text.MessageFormat; -import java.util.regex.Pattern; - -@Provider -@InterfaceAudience.Private -public class UserProvider extends AbstractHttpContextInjectable implements - InjectableProvider { - - public static final String USER_NAME_PARAM = "user.name"; - - - public static final String USER_PATTERN_KEY - = "httpfs.user.provider.user.pattern"; - - public static final String USER_PATTERN_DEFAULT - = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$"; - - private static Pattern userPattern = Pattern.compile(USER_PATTERN_DEFAULT); - - public static void setUserPattern(String pattern) { - userPattern = Pattern.compile(pattern); - } - - public static Pattern getUserPattern() { - return userPattern; - } - - static class UserParam extends StringParam { - - public UserParam(String user) { - super(USER_NAME_PARAM, user, getUserPattern()); - } - - @Override - public String parseParam(String str) { - if (str != null) { - int len = str.length(); - if (len < 1) { - throw new IllegalArgumentException(MessageFormat.format( - "Parameter [{0}], it's length must be at least 1", getName())); - } - } - return super.parseParam(str); - } - } - - @Override - public Principal getValue(HttpContext httpContext) { - Principal principal = httpContext.getRequest().getUserPrincipal(); - if (principal == null) { - final String user = httpContext.getRequest().getQueryParameters().getFirst(USER_NAME_PARAM); - if (user != null) { - principal = new Principal() { - @Override - public String getName() { - return new UserParam(user).value(); - } - }; - } - } - if (principal != null) { - MDC.put("user", principal.getName()); - } - return principal; - } - - @Override - public ComponentScope getScope() { - return ComponentScope.PerRequest; - } - - @Override - public Injectable getInjectable(ComponentContext componentContext, Context context, Type type) { - return (type.equals(Principal.class)) ? this : null; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml index 05e1400ca93..d28ffa5638a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml @@ -34,7 +34,6 @@ org.apache.hadoop.lib.service.instrumentation.InstrumentationService, org.apache.hadoop.lib.service.scheduler.SchedulerService, org.apache.hadoop.lib.service.security.GroupsService, - org.apache.hadoop.lib.service.security.ProxyUserService, org.apache.hadoop.lib.service.hadoop.FileSystemAccessService @@ -118,6 +117,10 @@ + diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestProxyUserService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestProxyUserService.java deleted file mode 100644 index 294f5e80b24..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestProxyUserService.java +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.service.security; - -import static org.junit.Assert.assertNotNull; - -import java.security.AccessControlException; -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.lib.server.Server; -import org.apache.hadoop.lib.server.ServiceException; -import org.apache.hadoop.lib.service.Groups; -import org.apache.hadoop.lib.service.ProxyUser; -import org.apache.hadoop.test.HTestCase; -import org.apache.hadoop.test.TestDir; -import org.apache.hadoop.test.TestDirHelper; -import org.apache.hadoop.test.TestException; -import org.apache.hadoop.util.StringUtils; -import org.junit.Test; - -public class TestProxyUserService extends HTestCase { - - @Test - @TestDir - public void service() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - server.destroy(); - } - - @Test - @TestException(exception = ServiceException.class, msgRegExp = "PRXU02.*") - @TestDir - public void wrongConfigGroups() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - } - - @Test - @TestException(exception = ServiceException.class, msgRegExp = "PRXU01.*") - @TestDir - public void wrongHost() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "otherhost"); - conf.set("server.proxyuser.foo.groups", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - } - - @Test - @TestException(exception = ServiceException.class, msgRegExp = "PRXU02.*") - @TestDir - public void wrongConfigHosts() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.groups", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - } - - @Test - @TestDir - public void validateAnyHostAnyUser() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "*"); - conf.set("server.proxyuser.foo.groups", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - proxyUser.validate("foo", "localhost", "bar"); - server.destroy(); - } - - @Test(expected = AccessControlException.class) - @TestDir - public void invalidProxyUser() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "*"); - conf.set("server.proxyuser.foo.groups", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - proxyUser.validate("bar", "localhost", "foo"); - server.destroy(); - } - - @Test - @TestDir - public void validateHost() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "localhost"); - conf.set("server.proxyuser.foo.groups", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - proxyUser.validate("foo", "localhost", "bar"); - server.destroy(); - } - - private String getGroup() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName()))); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - Groups groups = server.get(Groups.class); - List g = groups.getGroups(System.getProperty("user.name")); - server.destroy(); - return g.get(0); - } - - @Test - @TestDir - public void validateGroup() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "*"); - conf.set("server.proxyuser.foo.groups", getGroup()); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - proxyUser.validate("foo", "localhost", System.getProperty("user.name")); - server.destroy(); - } - - - @Test(expected = AccessControlException.class) - @TestDir - public void unknownHost() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "localhost"); - conf.set("server.proxyuser.foo.groups", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - proxyUser.validate("foo", "unknownhost.bar.foo", "bar"); - server.destroy(); - } - - @Test(expected = AccessControlException.class) - @TestDir - public void invalidHost() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "localhost"); - conf.set("server.proxyuser.foo.groups", "*"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - proxyUser.validate("foo", "www.yahoo.com", "bar"); - server.destroy(); - } - - @Test(expected = AccessControlException.class) - @TestDir - public void invalidGroup() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - Configuration conf = new Configuration(false); - conf.set("server.services", StringUtils.join(",", Arrays.asList(GroupsService.class.getName(), - ProxyUserService.class.getName()))); - conf.set("server.proxyuser.foo.hosts", "localhost"); - conf.set("server.proxyuser.foo.groups", "nobody"); - Server server = new Server("server", dir, dir, dir, dir, conf); - server.init(); - ProxyUser proxyUser = server.get(ProxyUser.class); - assertNotNull(proxyUser); - proxyUser.validate("foo", "localhost", System.getProperty("user.name")); - server.destroy(); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java deleted file mode 100644 index e8942fbe4ef..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/wsrs/TestUserProvider.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.lib.wsrs; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import java.security.Principal; - -import javax.ws.rs.core.MultivaluedMap; - -import org.apache.hadoop.test.TestException; -import org.apache.hadoop.test.TestExceptionHelper; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.MethodRule; -import org.mockito.Mockito; -import org.slf4j.MDC; - -import com.sun.jersey.api.core.HttpContext; -import com.sun.jersey.api.core.HttpRequestContext; -import com.sun.jersey.core.spi.component.ComponentScope; - -public class TestUserProvider { - - @Rule - public MethodRule exceptionHelper = new TestExceptionHelper(); - - @Test - @SuppressWarnings("unchecked") - public void noUser() { - MDC.remove("user"); - HttpRequestContext request = Mockito.mock(HttpRequestContext.class); - Mockito.when(request.getUserPrincipal()).thenReturn(null); - MultivaluedMap map = Mockito.mock(MultivaluedMap.class); - Mockito.when(map.getFirst(UserProvider.USER_NAME_PARAM)).thenReturn(null); - Mockito.when(request.getQueryParameters()).thenReturn(map); - HttpContext context = Mockito.mock(HttpContext.class); - Mockito.when(context.getRequest()).thenReturn(request); - UserProvider up = new UserProvider(); - assertNull(up.getValue(context)); - assertNull(MDC.get("user")); - } - - @Test - @SuppressWarnings("unchecked") - public void queryStringUser() { - MDC.remove("user"); - HttpRequestContext request = Mockito.mock(HttpRequestContext.class); - Mockito.when(request.getUserPrincipal()).thenReturn(null); - MultivaluedMap map = Mockito.mock(MultivaluedMap.class); - Mockito.when(map.getFirst(UserProvider.USER_NAME_PARAM)).thenReturn("foo"); - Mockito.when(request.getQueryParameters()).thenReturn(map); - HttpContext context = Mockito.mock(HttpContext.class); - Mockito.when(context.getRequest()).thenReturn(request); - UserProvider up = new UserProvider(); - assertEquals(up.getValue(context).getName(), "foo"); - assertEquals(MDC.get("user"), "foo"); - } - - @Test - @SuppressWarnings("unchecked") - public void principalUser() { - MDC.remove("user"); - HttpRequestContext request = Mockito.mock(HttpRequestContext.class); - Mockito.when(request.getUserPrincipal()).thenReturn(new Principal() { - @Override - public String getName() { - return "bar"; - } - }); - HttpContext context = Mockito.mock(HttpContext.class); - Mockito.when(context.getRequest()).thenReturn(request); - UserProvider up = new UserProvider(); - assertEquals(up.getValue(context).getName(), "bar"); - assertEquals(MDC.get("user"), "bar"); - } - - @Test - public void getters() { - UserProvider up = new UserProvider(); - assertEquals(up.getScope(), ComponentScope.PerRequest); - assertEquals(up.getInjectable(null, null, Principal.class), up); - assertNull(up.getInjectable(null, null, String.class)); - } - - @Test - @TestException(exception = IllegalArgumentException.class) - public void userNameEmpty() { - new UserProvider.UserParam(""); - } - - @Test - @TestException(exception = IllegalArgumentException.class) - public void userNameInvalidStart() { - new UserProvider.UserParam("1x"); - } - - @Test - @TestException(exception = IllegalArgumentException.class) - public void userNameInvalidDollarSign() { - new UserProvider.UserParam("1$x"); - } - - @Test - public void userNameMinLength() { - new UserProvider.UserParam("a"); - } - - @Test - public void userNameValidDollarSign() { - new UserProvider.UserParam("a$"); - } - - @Test - public void customUserPattern() { - try { - UserProvider.setUserPattern("1"); - new UserProvider.UserParam("1"); - } finally { - UserProvider.setUserPattern(UserProvider.USER_PATTERN_DEFAULT); - } - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9253b8b88a6..0a90370dc68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -399,6 +399,9 @@ Release 2.6.0 - UNRELEASED HDFS-6567. Normalize the order of public final in HdfsFileStatus. (Tassapol Athiapinya via wheat9) + HDFS-6849. Replace HttpFS custom proxyuser handling with common + implementation. (tucu) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) From 41aa4badf8270a8e7d4321809fe8b8a34db98968 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Wed, 13 Aug 2014 20:29:23 +0000 Subject: [PATCH 22/67] YARN-2277. Added cross-origin support for the timeline server web services. Contributed by Jonathan Eagles. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617832 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../timeline/webapp/CrossOriginFilter.java | 220 ++++++++++++++++++ .../webapp/CrossOriginFilterInitializer.java | 42 ++++ .../webapp/TestCrossOriginFilter.java | 214 +++++++++++++++++ .../TestCrossOriginFilterInitializer.java | 60 +++++ 5 files changed, 539 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 55a61126fe2..99b2d612229 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -41,6 +41,9 @@ Release 2.6.0 - UNRELEASED YARN-1337. Recover containers upon nodemanager restart. (Jason Lowe via junping_du) + YARN-2277. Added cross-origin support for the timeline server web services. + (Jonathan Eagles via zjshen) + IMPROVEMENTS YARN-2242. Improve exception information on AM launch crashes. (Li Lu diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java new file mode 100644 index 00000000000..a9fb3e875f5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.webapp; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.google.common.annotations.VisibleForTesting; + +public class CrossOriginFilter implements Filter { + + private static final Log LOG = LogFactory.getLog(CrossOriginFilter.class); + + // HTTP CORS Request Headers + static final String ORIGIN = "Origin"; + static final String ACCESS_CONTROL_REQUEST_METHOD = + "Access-Control-Request-Method"; + static final String ACCESS_CONTROL_REQUEST_HEADERS = + "Access-Control-Request-Headers"; + + // HTTP CORS Response Headers + static final String ACCESS_CONTROL_ALLOW_ORIGIN = + "Access-Control-Allow-Origin"; + static final String ACCESS_CONTROL_ALLOW_CREDENTIALS = + "Access-Control-Allow-Credentials"; + static final String ACCESS_CONTROL_ALLOW_METHODS = + "Access-Control-Allow-Methods"; + static final String ACCESS_CONTROL_ALLOW_HEADERS = + "Access-Control-Allow-Headers"; + static final String ACCESS_CONTROL_MAX_AGE = "Access-Control-Max-Age"; + + // Filter configuration + public static final String ALLOWED_ORIGINS = "allowed-origins"; + public static final String ALLOWED_ORIGINS_DEFAULT = "*"; + public static final String ALLOWED_METHODS = "allowed-methods"; + public static final String ALLOWED_METHODS_DEFAULT = "GET,POST,HEAD"; + public static final String ALLOWED_HEADERS = "allowed-headers"; + public static final String ALLOWED_HEADERS_DEFAULT = + "X-Requested-With,Content-Type,Accept,Origin"; + public static final String MAX_AGE = "max-age"; + public static final String MAX_AGE_DEFAULT = "1800"; + + private List allowedMethods = new ArrayList(); + private List allowedHeaders = new ArrayList(); + private List allowedOrigins = new ArrayList(); + private String maxAge; + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + initializeAllowedMethods(filterConfig); + initializeAllowedHeaders(filterConfig); + initializeAllowedOrigins(filterConfig); + initializeMaxAge(filterConfig); + } + + @Override + public void doFilter(ServletRequest req, ServletResponse res, + FilterChain chain) + throws IOException, ServletException { + doCrossFilter((HttpServletRequest) req, (HttpServletResponse) res); + chain.doFilter(req, res); + } + + @Override + public void destroy() { + allowedMethods.clear(); + allowedHeaders.clear(); + allowedOrigins.clear(); + } + + private void doCrossFilter(HttpServletRequest req, HttpServletResponse res) { + + String origin = encodeHeader(req.getHeader(ORIGIN)); + if (!isCrossOrigin(origin)) { + return; + } + + if (!isOriginAllowed(origin)) { + return; + } + + String accessControlRequestMethod = + req.getHeader(ACCESS_CONTROL_REQUEST_METHOD); + if (!isMethodAllowed(accessControlRequestMethod)) { + return; + } + + String accessControlRequestHeaders = + req.getHeader(ACCESS_CONTROL_REQUEST_HEADERS); + if (!areHeadersAllowed(accessControlRequestHeaders)) { + return; + } + + res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, origin); + res.setHeader(ACCESS_CONTROL_ALLOW_CREDENTIALS, Boolean.TRUE.toString()); + res.setHeader(ACCESS_CONTROL_ALLOW_METHODS, getAllowedMethodsHeader()); + res.setHeader(ACCESS_CONTROL_ALLOW_HEADERS, getAllowedHeadersHeader()); + res.setHeader(ACCESS_CONTROL_MAX_AGE, maxAge); + } + + @VisibleForTesting + String getAllowedHeadersHeader() { + return StringUtils.join(allowedHeaders, ','); + } + + @VisibleForTesting + String getAllowedMethodsHeader() { + return StringUtils.join(allowedMethods, ','); + } + + private void initializeAllowedMethods(FilterConfig filterConfig) { + String allowedMethodsConfig = + filterConfig.getInitParameter(ALLOWED_METHODS); + if (allowedMethodsConfig == null) { + allowedMethodsConfig = ALLOWED_METHODS_DEFAULT; + } + allowedMethods = + Arrays.asList(allowedMethodsConfig.trim().split("\\s*,\\s*")); + LOG.info("Allowed Methods: " + getAllowedMethodsHeader()); + } + + private void initializeAllowedHeaders(FilterConfig filterConfig) { + String allowedHeadersConfig = + filterConfig.getInitParameter(ALLOWED_HEADERS); + if (allowedHeadersConfig == null) { + allowedHeadersConfig = ALLOWED_HEADERS_DEFAULT; + } + allowedHeaders = + Arrays.asList(allowedHeadersConfig.trim().split("\\s*,\\s*")); + LOG.info("Allowed Headers: " + getAllowedHeadersHeader()); + } + + private void initializeAllowedOrigins(FilterConfig filterConfig) { + String allowedOriginsConfig = + filterConfig.getInitParameter(ALLOWED_ORIGINS); + if (allowedOriginsConfig == null) { + allowedOriginsConfig = ALLOWED_ORIGINS_DEFAULT; + } + allowedOrigins = + Arrays.asList(allowedOriginsConfig.trim().split("\\s*,\\s*")); + LOG.info("Allowed Origins: " + StringUtils.join(allowedOrigins, ',')); + } + + private void initializeMaxAge(FilterConfig filterConfig) { + maxAge = filterConfig.getInitParameter(MAX_AGE); + if (maxAge == null) { + maxAge = MAX_AGE_DEFAULT; + } + LOG.info("Max Age: " + maxAge); + } + + static String encodeHeader(final String header) { + if (header == null) { + return null; + } + try { + // Protect against HTTP response splitting vulnerability + // since value is written as part of the response header + return URLEncoder.encode(header, "ASCII"); + } catch (UnsupportedEncodingException e) { + return null; + } + } + + static boolean isCrossOrigin(String origin) { + return origin != null; + } + + private boolean isOriginAllowed(String origin) { + return allowedOrigins.contains(origin); + } + + private boolean areHeadersAllowed(String accessControlRequestHeaders) { + if (accessControlRequestHeaders == null) { + return true; + } + String headers[] = accessControlRequestHeaders.trim().split("\\s*,\\s*"); + return allowedHeaders.containsAll(Arrays.asList(headers)); + } + + private boolean isMethodAllowed(String accessControlRequestMethod) { + if (accessControlRequestMethod == null) { + return false; + } + return allowedMethods.contains(accessControlRequestMethod); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java new file mode 100644 index 00000000000..69e0188137a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.webapp; + +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.http.FilterInitializer; + +public class CrossOriginFilterInitializer extends FilterInitializer { + + public static final String PREFIX = + "yarn.timeline-service.http-cross-origin."; + + @Override + public void initFilter(FilterContainer container, Configuration conf) { + + container.addGlobalFilter("Cross Origin Filter", + CrossOriginFilter.class.getName(), getFilterParameters(conf)); + } + + static Map getFilterParameters(Configuration conf) { + return conf.getValByRegex(PREFIX); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java new file mode 100644 index 00000000000..a29e4a0aa6d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.webapp; + +import java.io.IOException; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.junit.Test; + +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; + +public class TestCrossOriginFilter { + + @Test + public void testSameOrigin() throws ServletException, IOException { + + // Setup the configuration settings of the server + Map conf = new HashMap(); + conf.put(CrossOriginFilter.ALLOWED_ORIGINS, ""); + FilterConfig filterConfig = new FilterConfigTest(conf); + + // Origin is not specified for same origin requests + HttpServletRequest mockReq = mock(HttpServletRequest.class); + when(mockReq.getHeader(CrossOriginFilter.ORIGIN)).thenReturn(null); + + // Objects to verify interactions based on request + HttpServletResponse mockRes = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + // Object under test + CrossOriginFilter filter = new CrossOriginFilter(); + filter.init(filterConfig); + filter.doFilter(mockReq, mockRes, mockChain); + + verifyZeroInteractions(mockRes); + verify(mockChain).doFilter(mockReq, mockRes); + } + + @Test + public void testDisallowedOrigin() throws ServletException, IOException { + + // Setup the configuration settings of the server + Map conf = new HashMap(); + conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "example.com"); + FilterConfig filterConfig = new FilterConfigTest(conf); + + // Origin is not specified for same origin requests + HttpServletRequest mockReq = mock(HttpServletRequest.class); + when(mockReq.getHeader(CrossOriginFilter.ORIGIN)).thenReturn("example.org"); + + // Objects to verify interactions based on request + HttpServletResponse mockRes = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + // Object under test + CrossOriginFilter filter = new CrossOriginFilter(); + filter.init(filterConfig); + filter.doFilter(mockReq, mockRes, mockChain); + + verifyZeroInteractions(mockRes); + verify(mockChain).doFilter(mockReq, mockRes); + } + + @Test + public void testDisallowedMethod() throws ServletException, IOException { + + // Setup the configuration settings of the server + Map conf = new HashMap(); + conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "example.com"); + FilterConfig filterConfig = new FilterConfigTest(conf); + + // Origin is not specified for same origin requests + HttpServletRequest mockReq = mock(HttpServletRequest.class); + when(mockReq.getHeader(CrossOriginFilter.ORIGIN)).thenReturn("example.com"); + when(mockReq.getHeader(CrossOriginFilter.ACCESS_CONTROL_REQUEST_METHOD)) + .thenReturn("DISALLOWED_METHOD"); + + // Objects to verify interactions based on request + HttpServletResponse mockRes = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + // Object under test + CrossOriginFilter filter = new CrossOriginFilter(); + filter.init(filterConfig); + filter.doFilter(mockReq, mockRes, mockChain); + + verifyZeroInteractions(mockRes); + verify(mockChain).doFilter(mockReq, mockRes); + } + + @Test + public void testDisallowedHeader() throws ServletException, IOException { + + // Setup the configuration settings of the server + Map conf = new HashMap(); + conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "example.com"); + FilterConfig filterConfig = new FilterConfigTest(conf); + + // Origin is not specified for same origin requests + HttpServletRequest mockReq = mock(HttpServletRequest.class); + when(mockReq.getHeader(CrossOriginFilter.ORIGIN)).thenReturn("example.com"); + when(mockReq.getHeader(CrossOriginFilter.ACCESS_CONTROL_REQUEST_METHOD)) + .thenReturn("GET"); + when(mockReq.getHeader(CrossOriginFilter.ACCESS_CONTROL_REQUEST_HEADERS)) + .thenReturn("Disallowed-Header"); + + // Objects to verify interactions based on request + HttpServletResponse mockRes = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + // Object under test + CrossOriginFilter filter = new CrossOriginFilter(); + filter.init(filterConfig); + filter.doFilter(mockReq, mockRes, mockChain); + + verifyZeroInteractions(mockRes); + verify(mockChain).doFilter(mockReq, mockRes); + } + + @Test + public void testCrossOriginFilter() throws ServletException, IOException { + + // Setup the configuration settings of the server + Map conf = new HashMap(); + conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "example.com"); + FilterConfig filterConfig = new FilterConfigTest(conf); + + // Origin is not specified for same origin requests + HttpServletRequest mockReq = mock(HttpServletRequest.class); + when(mockReq.getHeader(CrossOriginFilter.ORIGIN)).thenReturn("example.com"); + when(mockReq.getHeader(CrossOriginFilter.ACCESS_CONTROL_REQUEST_METHOD)) + .thenReturn("GET"); + when(mockReq.getHeader(CrossOriginFilter.ACCESS_CONTROL_REQUEST_HEADERS)) + .thenReturn("X-Requested-With"); + + // Objects to verify interactions based on request + HttpServletResponse mockRes = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + // Object under test + CrossOriginFilter filter = new CrossOriginFilter(); + filter.init(filterConfig); + filter.doFilter(mockReq, mockRes, mockChain); + + verify(mockRes).setHeader(CrossOriginFilter.ACCESS_CONTROL_ALLOW_ORIGIN, + "example.com"); + verify(mockRes).setHeader( + CrossOriginFilter.ACCESS_CONTROL_ALLOW_CREDENTIALS, + Boolean.TRUE.toString()); + verify(mockRes).setHeader(CrossOriginFilter.ACCESS_CONTROL_ALLOW_METHODS, + filter.getAllowedMethodsHeader()); + verify(mockRes).setHeader(CrossOriginFilter.ACCESS_CONTROL_ALLOW_HEADERS, + filter.getAllowedHeadersHeader()); + verify(mockChain).doFilter(mockReq, mockRes); + } + + private static class FilterConfigTest implements FilterConfig { + + final Map map; + + FilterConfigTest(Map map) { + this.map = map; + } + + @Override + public String getFilterName() { + return "test-filter"; + } + + @Override + public String getInitParameter(String key) { + return map.get(key); + } + + @Override + public Enumeration getInitParameterNames() { + return Collections.enumeration(map.keySet()); + } + + @Override + public ServletContext getServletContext() { + return null; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java new file mode 100644 index 00000000000..3199aac5089 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.webapp; + +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; + +import org.junit.Assert; +import org.junit.Test; + +public class TestCrossOriginFilterInitializer { + + @Test + public void testGetFilterParameters() { + + // Initialize configuration object + Configuration conf = new Configuration(); + conf.set(CrossOriginFilterInitializer.PREFIX + "rootparam", "rootvalue"); + conf.set(CrossOriginFilterInitializer.PREFIX + "nested.param", + "nestedvalue"); + conf.set("outofscopeparam", "outofscopevalue"); + + // call function under test + Map filterParameters = + CrossOriginFilterInitializer.getFilterParameters(conf); + + // retrieve values + String rootvalue = + filterParameters.get(CrossOriginFilterInitializer.PREFIX + "rootparam"); + String nestedvalue = + filterParameters.get(CrossOriginFilterInitializer.PREFIX + + "nested.param"); + String outofscopeparam = filterParameters.get("outofscopeparam"); + + // verify expected values are in place + Assert.assertEquals("Could not find filter parameter", "rootvalue", + rootvalue); + Assert.assertEquals("Could not find filter parameter", "nestedvalue", + nestedvalue); + Assert.assertNull("Found unexpected value in filter parameters", + outofscopeparam); + } +} From 1a535f8dcdfc4fab8ba6125680957caf55ea8f46 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Wed, 13 Aug 2014 21:07:18 +0000 Subject: [PATCH 23/67] YARN-2070. Made DistributedShell publish the short user name to the timeline server. Contributed by Robert Kanter. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617837 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../distributedshell/ApplicationMaster.java | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 99b2d612229..4b06e627fca 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -190,6 +190,9 @@ Release 2.6.0 - UNRELEASED YARN-2361. RMAppAttempt state machine entries for KILLED state has duplicate event entries. (Zhihai Xu via kasha) + YARN-2070. Made DistributedShell publish the short user name to the timeline + server. (Robert Kanter via zjshen) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 9051d31089f..4a842458691 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -1054,8 +1054,8 @@ public class ApplicationMaster { TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); - entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser() - .toString()); + entity.addPrimaryFilter("user", + UserGroupInformation.getCurrentUser().getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); @@ -1071,8 +1071,8 @@ public class ApplicationMaster { TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getContainerId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); - entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser() - .toString()); + entity.addPrimaryFilter("user", + UserGroupInformation.getCurrentUser().getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_END.toString()); @@ -1089,8 +1089,8 @@ public class ApplicationMaster { TimelineEntity entity = new TimelineEntity(); entity.setEntityId(appAttemptId); entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString()); - entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser() - .toString()); + entity.addPrimaryFilter("user", + UserGroupInformation.getCurrentUser().getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setEventType(appEvent.toString()); event.setTimestamp(System.currentTimeMillis()); From e90d940699f288ddfef5923f38cb619f6d6259f4 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 14 Aug 2014 01:36:12 +0000 Subject: [PATCH 24/67] HADOOP-10843. TestGridmixRecord unit tests failure on PowerPC (Jinghui Wang via Colin Patrick McCabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617860 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop/mapred/gridmix/TestGridmixRecord.java | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6a0fbc596e4..362c83e8f4b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -578,6 +578,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10966. Hadoop Common native compilation broken in windows. (David Villegas via Arpit Agarwal) + HADOOP-10843. TestGridmixRecord unit tests failure on PowerPC (Jinghui Wang + via Colin Patrick McCabe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java index 2f3ce701d6b..b3b72778f77 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixRecord.java @@ -140,10 +140,10 @@ public class TestGridmixRecord { final int chk = WritableComparator.compareBytes( out1.getData(), 0, out1.getLength(), out2.getData(), 0, out2.getLength()); - assertEquals(chk, x.compareTo(y)); - assertEquals(chk, cmp.compare( + assertEquals(Integer.signum(chk), Integer.signum(x.compareTo(y))); + assertEquals(Integer.signum(chk), Integer.signum(cmp.compare( out1.getData(), 0, out1.getLength(), - out2.getData(), 0, out2.getLength())); + out2.getData(), 0, out2.getLength()))); // write second copy, compare eq final int s1 = out1.getLength(); x.write(out1); @@ -153,8 +153,8 @@ public class TestGridmixRecord { y.write(out2); assertEquals(0, cmp.compare(out2.getData(), 0, s2, out2.getData(), s2, out2.getLength() - s2)); - assertEquals(chk, cmp.compare(out1.getData(), 0, s1, - out2.getData(), s2, out2.getLength() - s2)); + assertEquals(Integer.signum(chk), Integer.signum(cmp.compare(out1.getData(), 0, s1, + out2.getData(), s2, out2.getLength() - s2))); } } From 44864c68b59a72f27279212c1316adec37e5209a Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Thu, 14 Aug 2014 04:20:00 +0000 Subject: [PATCH 25/67] HDFS-6783. Fix HDFS CacheReplicationMonitor rescan logic. Contributed by Yi Liu and Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617872 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../CacheReplicationMonitor.java | 55 +++++++++++-------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0a90370dc68..cffa1679fd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -506,6 +506,8 @@ Release 2.6.0 - UNRELEASED HDFS-6247. Avoid timeouts for replaceBlock() call by sending intermediate responses to Balancer (vinayakumarb) + HDFS-6783. Fix HDFS CacheReplicationMonitor rescan logic. (Yi Liu via umamahesh) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index cf5a4a6504b..cfd6333469b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -103,22 +103,22 @@ public class CacheReplicationMonitor extends Thread implements Closeable { */ private final Condition scanFinished; - /** - * Whether there are pending CacheManager operations that necessitate a - * CacheReplicationMonitor rescan. Protected by the CRM lock. - */ - private boolean needsRescan = true; - - /** - * Whether we are currently doing a rescan. Protected by the CRM lock. - */ - private boolean isScanning = false; - /** * The number of rescans completed. Used to wait for scans to finish. * Protected by the CacheReplicationMonitor lock. */ - private long scanCount = 0; + private long completedScanCount = 0; + + /** + * The scan we're currently performing, or -1 if no scan is in progress. + * Protected by the CacheReplicationMonitor lock. + */ + private long curScanCount = -1; + + /** + * The number of rescans we need to complete. Protected by the CRM lock. + */ + private long neededScanCount = 0; /** * True if this monitor should terminate. Protected by the CRM lock. @@ -169,7 +169,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { LOG.info("Shutting down CacheReplicationMonitor"); return; } - if (needsRescan) { + if (completedScanCount < neededScanCount) { LOG.info("Rescanning because of pending operations"); break; } @@ -182,8 +182,6 @@ public class CacheReplicationMonitor extends Thread implements Closeable { doRescan.await(delta, TimeUnit.MILLISECONDS); curTimeMs = Time.monotonicNow(); } - isScanning = true; - needsRescan = false; } finally { lock.unlock(); } @@ -194,8 +192,8 @@ public class CacheReplicationMonitor extends Thread implements Closeable { // Update synchronization-related variables. lock.lock(); try { - isScanning = false; - scanCount++; + completedScanCount = curScanCount; + curScanCount = -1; scanFinished.signalAll(); } finally { lock.unlock(); @@ -226,16 +224,15 @@ public class CacheReplicationMonitor extends Thread implements Closeable { "Must not hold the FSN write lock when waiting for a rescan."); Preconditions.checkArgument(lock.isHeldByCurrentThread(), "Must hold the CRM lock when waiting for a rescan."); - if (!needsRescan) { + if (neededScanCount <= completedScanCount) { return; } // If no scan is already ongoing, mark the CRM as dirty and kick - if (!isScanning) { + if (curScanCount < 0) { doRescan.signal(); } // Wait until the scan finishes and the count advances - final long startCount = scanCount; - while ((!shutdown) && (startCount >= scanCount)) { + while ((!shutdown) && (completedScanCount < neededScanCount)) { try { scanFinished.await(); } catch (InterruptedException e) { @@ -253,7 +250,14 @@ public class CacheReplicationMonitor extends Thread implements Closeable { public void setNeedsRescan() { Preconditions.checkArgument(lock.isHeldByCurrentThread(), "Must hold the CRM lock when setting the needsRescan bit."); - this.needsRescan = true; + if (curScanCount >= 0) { + // If there is a scan in progress, we need to wait for the scan after + // that. + neededScanCount = curScanCount + 1; + } else { + // If there is no scan in progress, we need to wait for the next scan. + neededScanCount = completedScanCount + 1; + } } /** @@ -284,10 +288,17 @@ public class CacheReplicationMonitor extends Thread implements Closeable { scannedBlocks = 0; namesystem.writeLock(); try { + lock.lock(); if (shutdown) { throw new InterruptedException("CacheReplicationMonitor was " + "shut down."); } + curScanCount = completedScanCount + 1; + } + finally { + lock.unlock(); + } + try { resetStatistics(); rescanCacheDirectives(); rescanCachedBlockMap(); From bd79a4b9263726f4642c94bff689dbc109592a70 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 14 Aug 2014 04:29:35 +0000 Subject: [PATCH 26/67] HADOOP-10838. Byte array native checksumming. Contributed by James Thomas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617875 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../org/apache/hadoop/util/DataChecksum.java | 6 ++ .../org/apache/hadoop/util/NativeCrc32.java | 16 ++++ .../src/org/apache/hadoop/util/NativeCrc32.c | 82 +++++++++++++++++++ 4 files changed, 106 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 362c83e8f4b..9215927bb82 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -513,6 +513,8 @@ Release 2.6.0 - UNRELEASED OPTIMIZATIONS + HADOOP-10838. Byte array native checksumming. (James Thomas via todd) + BUG FIXES HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index c50ecfd64ed..27984b48c7e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -339,6 +339,12 @@ public class DataChecksum implements Checksum { byte[] data, int dataOff, int dataLen, byte[] checksums, int checksumsOff, String fileName, long basePos) throws ChecksumException { + + if (NativeCrc32.isAvailable()) { + NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, type.id, + checksums, checksumsOff, data, dataOff, dataLen, fileName, basePos); + return; + } int remaining = dataLen; int dataPos = 0; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java index 13b2c9a5816..26a5e9b10dd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java @@ -59,6 +59,16 @@ class NativeCrc32 { data, data.position(), data.remaining(), fileName, basePos); } + + public static void verifyChunkedSumsByteArray(int bytesPerSum, + int checksumType, byte[] sums, int sumsOffset, byte[] data, + int dataOffset, int dataLength, String fileName, long basePos) + throws ChecksumException { + nativeVerifyChunkedSumsByteArray(bytesPerSum, checksumType, + sums, sumsOffset, + data, dataOffset, dataLength, + fileName, basePos); + } private static native void nativeVerifyChunkedSums( int bytesPerSum, int checksumType, @@ -66,6 +76,12 @@ class NativeCrc32 { ByteBuffer data, int dataOffset, int dataLength, String fileName, long basePos); + private static native void nativeVerifyChunkedSumsByteArray( + int bytesPerSum, int checksumType, + byte[] sums, int sumsOffset, + byte[] data, int dataOffset, int dataLength, + String fileName, long basePos); + // Copy the constants over from DataChecksum so that javah will pick them up // and make them available in the native code header. public static final int CHECKSUM_CRC32 = DataChecksum.CHECKSUM_CRC32; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c index cba25fa3047..4bd69bc41d7 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -34,6 +34,10 @@ #include "bulk_crc32.h" +#define MBYTE 1048576 +#define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) +#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) + static void throw_checksum_exception(JNIEnv *env, uint32_t got_crc, uint32_t expected_crc, jstring j_filename, jlong pos) { @@ -177,6 +181,84 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk } } +JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunkedSumsByteArray + (JNIEnv *env, jclass clazz, + jint bytes_per_checksum, jint j_crc_type, + jarray j_sums, jint sums_offset, + jarray j_data, jint data_offset, jint data_len, + jstring j_filename, jlong base_pos) +{ + uint8_t *sums_addr; + uint8_t *data_addr; + uint32_t *sums; + uint8_t *data; + int crc_type; + crc32_error_t error_data; + int ret; + int numChecksumsPerIter; + int checksumNum; + + if (unlikely(!j_sums || !j_data)) { + THROW(env, "java/lang/NullPointerException", + "input byte arrays must not be null"); + return; + } + if (unlikely(sums_offset < 0 || data_offset < 0 || data_len < 0)) { + THROW(env, "java/lang/IllegalArgumentException", + "bad offsets or lengths"); + return; + } + if (unlikely(bytes_per_checksum) <= 0) { + THROW(env, "java/lang/IllegalArgumentException", + "invalid bytes_per_checksum"); + return; + } + + // Convert to correct internal C constant for CRC type + crc_type = convert_java_crc_type(env, j_crc_type); + if (crc_type == -1) return; // exception already thrown + + numChecksumsPerIter = MAX(1, MBYTE / bytes_per_checksum); + checksumNum = 0; + while (checksumNum * bytes_per_checksum < data_len) { + // Convert byte arrays to C pointers + sums_addr = (*env)->GetPrimitiveArrayCritical(env, j_sums, NULL); + data_addr = (*env)->GetPrimitiveArrayCritical(env, j_data, NULL); + + if (unlikely(!sums_addr || !data_addr)) { + if (data_addr) (*env)->ReleasePrimitiveArrayCritical(env, j_data, data_addr, 0); + if (sums_addr) (*env)->ReleasePrimitiveArrayCritical(env, j_sums, sums_addr, 0); + THROW(env, "java/lang/OutOfMemoryError", + "not enough memory for byte arrays in JNI code"); + return; + } + + sums = (uint32_t *)(sums_addr + sums_offset) + checksumNum; + data = data_addr + data_offset + checksumNum * bytes_per_checksum; + + // Setup complete. Actually verify checksums. + ret = bulk_verify_crc(data, MIN(numChecksumsPerIter * bytes_per_checksum, + data_len - checksumNum * bytes_per_checksum), + sums, crc_type, bytes_per_checksum, &error_data); + (*env)->ReleasePrimitiveArrayCritical(env, j_data, data_addr, 0); + (*env)->ReleasePrimitiveArrayCritical(env, j_sums, sums_addr, 0); + if (unlikely(ret == INVALID_CHECKSUM_DETECTED)) { + long pos = base_pos + (error_data.bad_data - data) + checksumNum * + bytes_per_checksum; + throw_checksum_exception( + env, error_data.got_crc, error_data.expected_crc, + j_filename, pos); + return; + } else if (unlikely(ret != CHECKSUMS_VALID)) { + THROW(env, "java/lang/AssertionError", + "Bad response code from native bulk_verify_crc"); + return; + } + checksumNum += numChecksumsPerIter; + } + +} + /** * vim: sw=2: ts=2: et: */ From 13ad623b9dbb479d3aed4ef3919e15da36c38fec Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Thu, 14 Aug 2014 04:57:47 +0000 Subject: [PATCH 27/67] Updated contributors name in CHANGES.txt for HDFS-6783 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617880 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cffa1679fd2..393938ce4ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -506,7 +506,7 @@ Release 2.6.0 - UNRELEASED HDFS-6247. Avoid timeouts for replaceBlock() call by sending intermediate responses to Balancer (vinayakumarb) - HDFS-6783. Fix HDFS CacheReplicationMonitor rescan logic. (Yi Liu via umamahesh) + HDFS-6783. Fix HDFS CacheReplicationMonitor rescan logic. (Yi Liu and Colin Patrick McCabe via umamahesh) Release 2.5.0 - UNRELEASED From b3df973adb6af68a881418151a903cb205392137 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 14 Aug 2014 15:39:52 +0000 Subject: [PATCH 28/67] MAPREDUCE-883. harchive: Document how to unarchive (Akira AJISAKA and Koji Noguchi via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617977 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/site/markdown/HadoopArchives.md.vm | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 22ca1b54e9b..dd3a2d02e45 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -86,6 +86,9 @@ Trunk (Unreleased) MAPREDUCE-6019. MapReduce changes for exposing YARN/MR endpoints on multiple interfaces. (Craig Welch, Milan Potocnik, Arpit Agarwal via xgong) + MAPREDUCE-883. harchive: Document how to unarchive (Akira AJISAKA and + Koji Noguchi via aw) + BUG FIXES MAPREDUCE-5714. Removed forceful JVM exit in shutDownJob. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm index 431310a9f5c..0cc0f1c93aa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm @@ -20,6 +20,7 @@ Hadoop Archives Guide - [Overview](#Overview) - [How to Create an Archive](#How_to_Create_an_Archive) - [How to Look Up Files in Archives](#How_to_Look_Up_Files_in_Archives) + - [How to Unarchive an Archive](#How_to_Unarchive_an_Archive) - [Archives Examples](#Archives_Examples) - [Creating an Archive](#Creating_an_Archive) - [Looking Up Files](#Looking_Up_Files) @@ -70,6 +71,20 @@ How to Look Up Files in Archives `har:///archivepath/fileinarchive` +How to Unarchive an Archive +--------------------------- + + Since all the fs shell commands in the archives work transparently, + unarchiving is just a matter of copying. + + To unarchive sequentially: + + `hdfs dfs -cp har:///user/zoo/foo.har/dir1 hdfs:/user/zoo/newdir` + + To unarchive in parallel, use DistCp: + + `hadoop distcp har:///user/zoo/foo.har/dir1 hdfs:/user/zoo/newdir` + Archives Examples ----------------- From 1e016e6451c1c8359ade42f91d8533abdedffd65 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 14 Aug 2014 15:43:28 +0000 Subject: [PATCH 29/67] MAPREDUCE-4791. Javadoc for KeyValueTextInputFormat should include default separator and how to change it (Akira AJISAKA via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617979 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/lib/input/KeyValueTextInputFormat.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dd3a2d02e45..049c8b89dcf 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -89,6 +89,9 @@ Trunk (Unreleased) MAPREDUCE-883. harchive: Document how to unarchive (Akira AJISAKA and Koji Noguchi via aw) + MAPREDUCE-4791. Javadoc for KeyValueTextInputFormat should include default + separator and how to change it (Akira AJISAKA via aw) + BUG FIXES MAPREDUCE-5714. Removed forceful JVM exit in shutDownJob. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueTextInputFormat.java index 336c960910d..0800a0effa6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueTextInputFormat.java @@ -38,6 +38,9 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; * Either line feed or carriage-return are used to signal end of line. * Each line is divided into key and value parts by a separator byte. If no * such a byte exists, the key will be the entire line and value will be empty. + * The separator byte can be specified in config file under the attribute name + * mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default + * is the tab character ('\t'). */ @InterfaceAudience.Public @InterfaceStability.Stable From e7b540eced929a9c3a3ca6aab6c40571815a23de Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 14 Aug 2014 15:49:10 +0000 Subject: [PATCH 30/67] MAPREDUCE-5363. Fix doc and spelling for TaskCompletionEvent#getTaskStatus and getStatus (Akira AJISAKA via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617982 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/mapred/TaskCompletionEvent.java | 4 ++-- .../java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 049c8b89dcf..bb3d67a8363 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -163,6 +163,9 @@ Trunk (Unreleased) MAPREDUCE-5943. Separate mapred commands from CommandManual.apt.vm (Akira AJISAKA via aw) + MAPREDUCE-5363. Fix doc and spelling for TaskCompletionEvent#getTaskStatus + and getStatus (Akira AJISAKA via aw) + Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java index 742da3f86a4..dc4d82e116c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java @@ -90,8 +90,8 @@ public class TaskCompletionEvent } /** - * Returns enum Status.SUCESS or Status.FAILURE. - * @return task tracker status + * Returns {@link Status} + * @return task completion status */ public Status getTaskStatus() { return Status.valueOf(super.getStatus().name()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java index 9118f62e01c..31643a94ac2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java @@ -95,8 +95,8 @@ public class TaskCompletionEvent implements Writable{ } /** - * Returns enum Status.SUCESS or Status.FAILURE. - * @return task tracker status + * Returns {@link Status} + * @return task completion status */ public Status getStatus() { return status; From 9579554988f82d506a32b81834f3a4fa9c698471 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Thu, 14 Aug 2014 15:55:03 +0000 Subject: [PATCH 31/67] MAPREDUCE-6010. HistoryServerFileSystemStateStore fails to update tokens. Contributed by Jason Lowe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617984 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + ...toryServerFileSystemStateStoreService.java | 101 ++++++++++++++---- ...toryServerFileSystemStateStoreService.java | 85 ++++++++++++++- 3 files changed, 166 insertions(+), 23 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bb3d67a8363..17bf08cd49c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -202,6 +202,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6021. MR AM should have working directory in LD_LIBRARY_PATH (jlowe) + MAPREDUCE-6010. HistoryServerFileSystemStateStore fails to update tokens + (jlowe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java index d92a9af4b07..dcea333b5f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java @@ -24,6 +24,8 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.HashSet; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -64,6 +66,7 @@ public class HistoryServerFileSystemStateStoreService private static final String TOKEN_MASTER_KEY_FILE_PREFIX = "key_"; private static final String TOKEN_FILE_PREFIX = "token_"; private static final String TMP_FILE_PREFIX = "tmp-"; + private static final String UPDATE_TMP_FILE_PREFIX = "update-"; private static final FsPermission DIR_PERMISSIONS = new FsPermission((short)0700); private static final FsPermission FILE_PERMISSIONS = Shell.WINDOWS @@ -90,7 +93,7 @@ public class HistoryServerFileSystemStateStoreService @Override protected void startStorage() throws IOException { - fs = rootStatePath.getFileSystem(getConfig()); + fs = createFileSystem(); createDir(rootStatePath); tokenStatePath = new Path(rootStatePath, TOKEN_STATE_DIR_NAME); createDir(tokenStatePath); @@ -101,6 +104,10 @@ public class HistoryServerFileSystemStateStoreService } } + FileSystem createFileSystem() throws IOException { + return rootStatePath.getFileSystem(getConfig()); + } + @Override protected void closeStorage() throws IOException { // don't close the filesystem as it's part of the filesystem cache @@ -127,7 +134,7 @@ public class HistoryServerFileSystemStateStoreService throw new IOException(tokenPath + " already exists"); } - createFile(tokenPath, buildTokenData(tokenId, renewDate)); + createNewFile(tokenPath, buildTokenData(tokenId, renewDate)); } @Override @@ -136,7 +143,25 @@ public class HistoryServerFileSystemStateStoreService if (LOG.isDebugEnabled()) { LOG.debug("Updating token " + tokenId.getSequenceNumber()); } - createFile(getTokenPath(tokenId), buildTokenData(tokenId, renewDate)); + + // Files cannot be atomically replaced, therefore we write a temporary + // update file, remove the original token file, then rename the update + // file to the token file. During recovery either the token file will be + // used or if that is missing and an update file is present then the + // update file is used. + Path tokenPath = getTokenPath(tokenId); + Path tmp = new Path(tokenPath.getParent(), + UPDATE_TMP_FILE_PREFIX + tokenPath.getName()); + writeFile(tmp, buildTokenData(tokenId, renewDate)); + try { + deleteFile(tokenPath); + } catch (IOException e) { + fs.delete(tmp, false); + throw e; + } + if (!fs.rename(tmp, tokenPath)) { + throw new IOException("Could not rename " + tmp + " to " + tokenPath); + } } @Override @@ -168,7 +193,7 @@ public class HistoryServerFileSystemStateStoreService IOUtils.cleanup(LOG, dataStream); } - createFile(keyPath, memStream.toByteArray()); + createNewFile(keyPath, memStream.toByteArray()); } @Override @@ -213,23 +238,33 @@ public class HistoryServerFileSystemStateStoreService } } - private void createFile(Path file, byte[] data) throws IOException { - final int WRITE_BUFFER_SIZE = 4096; + private void createNewFile(Path file, byte[] data) + throws IOException { Path tmp = new Path(file.getParent(), TMP_FILE_PREFIX + file.getName()); - FSDataOutputStream out = fs.create(tmp, FILE_PERMISSIONS, true, - WRITE_BUFFER_SIZE, fs.getDefaultReplication(tmp), - fs.getDefaultBlockSize(tmp), null); + writeFile(tmp, data); + try { + if (!fs.rename(tmp, file)) { + throw new IOException("Could not rename " + tmp + " to " + file); + } + } catch (IOException e) { + fs.delete(tmp, false); + throw e; + } + } + + private void writeFile(Path file, byte[] data) throws IOException { + final int WRITE_BUFFER_SIZE = 4096; + FSDataOutputStream out = fs.create(file, FILE_PERMISSIONS, true, + WRITE_BUFFER_SIZE, fs.getDefaultReplication(file), + fs.getDefaultBlockSize(file), null); try { try { out.write(data); } finally { IOUtils.cleanup(LOG, out); } - if (!fs.rename(tmp, file)) { - throw new IOException("Could not rename " + tmp + " to " + file); - } } catch (IOException e) { - fs.delete(tmp, false); + fs.delete(file, false); throw e; } } @@ -284,6 +319,19 @@ public class HistoryServerFileSystemStateStoreService state.tokenMasterKeyState.add(key); } + private void loadTokenFromBucket(int bucketId, + HistoryServerState state, Path tokenFile, long numTokenFileBytes) + throws IOException { + MRDelegationTokenIdentifier token = + loadToken(state, tokenFile, numTokenFileBytes); + int tokenBucketId = getBucketId(token); + if (tokenBucketId != bucketId) { + throw new IOException("Token " + tokenFile + + " should be in bucket " + tokenBucketId + ", found in bucket " + + bucketId); + } + } + private MRDelegationTokenIdentifier loadToken(HistoryServerState state, Path tokenFile, long numTokenFileBytes) throws IOException { MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier(); @@ -308,18 +356,29 @@ public class HistoryServerFileSystemStateStoreService final int bucketId = Integer.parseInt(numStr); int numTokens = 0; FileStatus[] tokenStats = fs.listStatus(bucket); + Set loadedTokens = new HashSet(tokenStats.length); for (FileStatus stat : tokenStats) { String name = stat.getPath().getName(); if (name.startsWith(TOKEN_FILE_PREFIX)) { - MRDelegationTokenIdentifier token = - loadToken(state, stat.getPath(), stat.getLen()); - int tokenBucketId = getBucketId(token); - if (tokenBucketId != bucketId) { - throw new IOException("Token " + stat.getPath() - + " should be in bucket " + tokenBucketId + ", found in bucket " - + bucketId); - } + loadTokenFromBucket(bucketId, state, stat.getPath(), stat.getLen()); + loadedTokens.add(name); ++numTokens; + } else if (name.startsWith(UPDATE_TMP_FILE_PREFIX)) { + String tokenName = name.substring(UPDATE_TMP_FILE_PREFIX.length()); + if (loadedTokens.contains(tokenName)) { + // already have the token, update may be partial so ignore it + fs.delete(stat.getPath(), false); + } else { + // token is missing, so try to parse the update temp file + loadTokenFromBucket(bucketId, state, stat.getPath(), stat.getLen()); + fs.rename(stat.getPath(), + new Path(stat.getPath().getParent(), tokenName)); + loadedTokens.add(tokenName); + ++numTokens; + } + } else if (name.startsWith(TMP_FILE_PREFIX)) { + // cleanup incomplete temp files + fs.delete(stat.getPath(), false); } else { LOG.warn("Skipping unexpected file in history server token bucket: " + stat.getPath()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryServerFileSystemStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryServerFileSystemStateStoreService.java index c9159963310..0a790028a16 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryServerFileSystemStateStoreService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryServerFileSystemStateStoreService.java @@ -21,12 +21,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.argThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.isA; +import static org.mockito.Mockito.spy; import java.io.File; import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier; import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService.HistoryServerState; @@ -35,6 +42,7 @@ import org.apache.hadoop.security.token.delegation.DelegationKey; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentMatcher; public class TestHistoryServerFileSystemStateStoreService { @@ -74,8 +82,8 @@ public class TestHistoryServerFileSystemStateStoreService { return store; } - @Test - public void testTokenStore() throws IOException { + private void testTokenStore(String stateStoreUri) throws IOException { + conf.set(JHAdminConfig.MR_HS_FS_STATE_STORE_URI, stateStoreUri); HistoryServerStateStoreService store = createAndStartStore(); HistoryServerState state = store.loadState(); @@ -161,4 +169,77 @@ public class TestHistoryServerFileSystemStateStoreService { assertTrue("missing master key 3", state.tokenMasterKeyState.contains(key3)); } + + @Test + public void testTokenStore() throws IOException { + testTokenStore(testDir.getAbsoluteFile().toURI().toString()); + } + + @Test + public void testTokenStoreHdfs() throws IOException { + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + conf = cluster.getConfiguration(0); + try { + testTokenStore("/tmp/historystore"); + } finally { + cluster.shutdown(); + } + } + + @Test + public void testUpdatedTokenRecovery() throws IOException { + IOException intentionalErr = new IOException("intentional error"); + FileSystem fs = FileSystem.getLocal(conf); + final FileSystem spyfs = spy(fs); + // make the update token process fail halfway through where we're left + // with just the temporary update file and no token file + ArgumentMatcher updateTmpMatcher = new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + if (argument instanceof Path) { + return ((Path) argument).getName().startsWith("update"); + } + return false; + } + }; + doThrow(intentionalErr) + .when(spyfs).rename(argThat(updateTmpMatcher), isA(Path.class)); + + conf.set(JHAdminConfig.MR_HS_FS_STATE_STORE_URI, + testDir.getAbsoluteFile().toURI().toString()); + HistoryServerStateStoreService store = + new HistoryServerFileSystemStateStoreService() { + @Override + FileSystem createFileSystem() throws IOException { + return spyfs; + } + }; + store.init(conf); + store.start(); + + final MRDelegationTokenIdentifier token1 = + new MRDelegationTokenIdentifier(new Text("tokenOwner1"), + new Text("tokenRenewer1"), new Text("tokenUser1")); + token1.setSequenceNumber(1); + final Long tokenDate1 = 1L; + store.storeToken(token1, tokenDate1); + final Long newTokenDate1 = 975318642L; + try { + store.updateToken(token1, newTokenDate1); + fail("intentional error not thrown"); + } catch (IOException e) { + assertEquals(intentionalErr, e); + } + store.close(); + + // verify the update file is seen and parsed upon recovery when + // original token file is missing + store = createAndStartStore(); + HistoryServerState state = store.loadState(); + assertEquals("incorrect loaded token count", 1, state.tokenState.size()); + assertTrue("missing token 1", state.tokenState.containsKey(token1)); + assertEquals("incorrect token 1 date", newTokenDate1, + state.tokenState.get(token1)); + store.close(); + } } From 92054f2a06a017d3c643d3b78e40483579739fca Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 14 Aug 2014 16:00:18 +0000 Subject: [PATCH 32/67] MAPREDUCE-5597. Missing alternatives in javadocs for deprecated constructors in mapreduce.Job (Akira AJISAKA via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617986 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 5 +++++ .../main/java/org/apache/hadoop/mapreduce/Job.java | 11 ++++++++++- .../mapreduce/task/reduce/MergeManagerImpl.java | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 17bf08cd49c..6d1594cad88 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -166,6 +166,11 @@ Trunk (Unreleased) MAPREDUCE-5363. Fix doc and spelling for TaskCompletionEvent#getTaskStatus and getStatus (Akira AJISAKA via aw) + MAPREDUCE-5595. Typo in MergeManagerImpl.java (Akira AJISAKA via aw) + + MAPREDUCE-5597. Missing alternatives in javadocs for deprecated constructors + in mapreduce.Job (Akira AJISAKA via aw) + Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java index 115a2b9fee3..3f8d13928ca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java @@ -54,7 +54,7 @@ import org.apache.hadoop.util.StringUtils; *

Here is an example on how to submit a job:

* 

  *     // Create a new Job
- *     Job job = new Job(new Configuration());
+ *     Job job = Job.getInstance();
  *     job.setJarByClass(MyJob.class);
  *     
  *     // Specify various job-specific parameters     
@@ -113,16 +113,25 @@ public class Job extends JobContextImpl implements JobContext {
   private long statustime;
   private Cluster cluster;
 
+  /**
+   * @deprecated Use {@link #getInstance()}
+   */
   @Deprecated
   public Job() throws IOException {
     this(new Configuration());
   }
 
+  /**
+   * @deprecated Use {@link #getInstance(Configuration)}
+   */
   @Deprecated
   public Job(Configuration conf) throws IOException {
     this(new JobConf(conf));
   }
 
+  /**
+   * @deprecated Use {@link #getInstance(Configuration, String)}
+   */
   @Deprecated
   public Job(Configuration conf, String jobName) throws IOException {
     this(conf);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
index a821e4d1b8a..4d83995dcfd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -197,7 +197,7 @@ public class MergeManagerImpl implements MergeManager {
              "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold);
 
     if (this.maxSingleShuffleLimit >= this.mergeThreshold) {
-      throw new RuntimeException("Invlaid configuration: "
+      throw new RuntimeException("Invalid configuration: "
           + "maxSingleShuffleLimit should be less than mergeThreshold"
           + "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit
           + "mergeThreshold: " + this.mergeThreshold);

From 6be98091f7636bb029130ebdffc1e9a26c707798 Mon Sep 17 00:00:00 2001
From: Allen Wittenauer 
Date: Thu, 14 Aug 2014 17:11:39 +0000
Subject: [PATCH 33/67] MAPREDUCE-5950. incorrect description in distcp2
 document (Akira AJISAKA via aw)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617994 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-mapreduce-project/CHANGES.txt                      | 3 +++
 .../src/site/markdown/DistCp.md.vm                        | 8 ++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 6d1594cad88..f2a99b6ef07 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -171,6 +171,9 @@ Trunk (Unreleased)
     MAPREDUCE-5597. Missing alternatives in javadocs for deprecated constructors 
       in mapreduce.Job (Akira AJISAKA via aw)
 
+    MAPREDUCE-5950. incorrect description in distcp2 document (Akira AJISAKA 
+      via aw)
+
 Release 2.6.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
index 6271a927a47..41b381ab6ec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
@@ -118,9 +118,9 @@ $H3 Basic Usage
 
 $H3 Update and Overwrite
 
-  `-update` is used to copy files from source that don't exist at the target,
-  or have different contents. `-overwrite` overwrites target-files even if they
-  exist at the source, or have the same contents.
+  `-update` is used to copy files from source that don't exist at the target
+  or differ than the target version. `-overwrite` overwrites target-files that
+  exist at the target.
 
   Update and Overwrite options warrant special attention, since their handling
   of source-paths varies from the defaults in a very subtle manner. Consider a
@@ -201,7 +201,7 @@ Flag              | Description                          | Notes
 `-log ` | Write logs to \ | DistCp keeps logs of each file it attempts to copy as map output. If a map fails, the log output will not be retained if it is re-executed.
 `-m ` | Maximum number of simultaneous copies | Specify the number of maps to copy data. Note that more maps may not necessarily improve throughput.
 `-overwrite` | Overwrite destination | If a map fails and `-i` is not specified, all the files in the split, not only those that failed, will be recopied. As discussed in the Usage documentation, it also changes the semantics for generating destination paths, so users should use this carefully.
-`-update` | Overwrite if src size different from dst size | As noted in the preceding, this is not a "sync" operation. The only criterion examined is the source and destination file sizes; if they differ, the source file replaces the destination file. As discussed in the Usage documentation, it also changes the semantics for generating destination paths, so users should use this carefully.
+`-update` | Overwrite if source and destination differ in size, blocksize, or checksum | As noted in the preceding, this is not a "sync" operation. The criteria examined are the source and destination file sizes, blocksizes, and checksums; if they differ, the source file replaces the destination file. As discussed in the Usage documentation, it also changes the semantics for generating destination paths, so users should use this carefully.
 `-f ` | Use list at \ as src list | This is equivalent to listing each source on the command line. The `urilist_uri` list should be a fully qualified URI.
 `-filelimit ` | Limit the total number of files to be <= n | **Deprecated!** Ignored in the new DistCp.
 `-sizelimit ` | Limit the total size to be <= n bytes | **Deprecated!** Ignored in the new DistCp.

From 7557f7bb2d22f564e26d6cf9672d24b7ef4f347a Mon Sep 17 00:00:00 2001
From: Allen Wittenauer 
Date: Thu, 14 Aug 2014 17:18:14 +0000
Subject: [PATCH 34/67] MAPREDUCE-5998. CompositeInputFormat javadoc is broken
 (Akira AJISAKA via aw)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1617995 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-mapreduce-project/CHANGES.txt                           | 3 +++
 .../org/apache/hadoop/mapred/join/CompositeInputFormat.java    | 2 +-
 .../apache/hadoop/mapreduce/lib/join/CompositeInputFormat.java | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index f2a99b6ef07..d3a26dbceef 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -174,6 +174,9 @@ Trunk (Unreleased)
     MAPREDUCE-5950. incorrect description in distcp2 document (Akira AJISAKA 
       via aw)
 
+    MAPREDUCE-5998. CompositeInputFormat javadoc is broken (Akira AJISAKA via
+      aw)
+
 Release 2.6.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java
index b177e8b8c00..40690e7541f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.mapred.Reporter;
 /**
  * An InputFormat capable of performing joins over a set of data sources sorted
  * and partitioned the same way.
- * @see #setFormat
  *
  * A user may define new join types by setting the property
  * mapred.join.define.<ident> to a classname. In the expression
@@ -44,6 +43,7 @@ import org.apache.hadoop.mapred.Reporter;
  * ComposableRecordReader.
  * mapred.join.keycomparator can be a classname used to compare keys
  * in the join.
+ * @see #setFormat
  * @see JoinRecordReader
  * @see MultiFilterRecordReader
  */
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputFormat.java
index bfb4bf14045..6189a271bc3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputFormat.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
 /**
  * An InputFormat capable of performing joins over a set of data sources sorted
  * and partitioned the same way.
- * @see #setFormat
  *
  * A user may define new join types by setting the property
  * mapreduce.join.define.<ident> to a classname. 
@@ -47,6 +46,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
  * assumed to be a ComposableRecordReader.
  * mapreduce.join.keycomparator can be a classname used to compare 
  * keys in the join.
+ * @see #setFormat
  * @see JoinRecordReader
  * @see MultiFilterRecordReader
  */

From e1f15d2cdf6be26df2716fee3c325895deedfc2f Mon Sep 17 00:00:00 2001
From: Allen Wittenauer 
Date: Thu, 14 Aug 2014 17:56:53 +0000
Subject: [PATCH 35/67] MAPREDUCE-5999. Fix dead link in InputFormat javadoc
 (Akira AJISAKA via aw)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618003 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-mapreduce-project/CHANGES.txt                            | 2 ++
 .../src/main/java/org/apache/hadoop/mapred/InputFormat.java     | 2 +-
 .../src/main/java/org/apache/hadoop/mapreduce/InputFormat.java  | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index d3a26dbceef..a44f2e41ada 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -177,6 +177,8 @@ Trunk (Unreleased)
     MAPREDUCE-5998. CompositeInputFormat javadoc is broken (Akira AJISAKA via
       aw)
 
+    MAPREDUCE-5999. Fix dead link in InputFormat javadoc (Akira AJISAKA via aw)
+
 Release 2.6.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputFormat.java
index afebdcf60b5..6f7b27893a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputFormat.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.fs.FileSystem;
  * bytes, of the input files. However, the {@link FileSystem} blocksize of  
  * the input files is treated as an upper bound for input splits. A lower bound 
  * on the split size can be set via 
- * 
+ * 
  * mapreduce.input.fileinputformat.split.minsize.

  * 
  * 
Clearly, logical splits based on input-size is insufficient for many 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputFormat.java
index 83559904f42..37648b748d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputFormat.java
@@ -52,7 +52,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  * bytes, of the input files. However, the {@link FileSystem} blocksize of  
  * the input files is treated as an upper bound for input splits. A lower bound 
  * on the split size can be set via 
- * 
+ * 
  * mapreduce.input.fileinputformat.split.minsize.

  * 
  * 
Clearly, logical splits based on input-size is insufficient for many 

From 2d904b3fa3bb406415f6e7b9807fcd5afa564f04 Mon Sep 17 00:00:00 2001
From: Allen Wittenauer 
Date: Thu, 14 Aug 2014 18:07:06 +0000
Subject: [PATCH 36/67] MAPREDUCE-5906. Inconsistent configuration in property
 "mapreduce.reduce.shuffle.input.buffer.percent" (Akira AJISAKA via aw)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618007 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-mapreduce-project/CHANGES.txt                           | 3 +++
 .../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 1 +
 .../apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java  | 3 ++-
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a44f2e41ada..7dc4436a887 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -92,6 +92,9 @@ Trunk (Unreleased)
     MAPREDUCE-4791. Javadoc for KeyValueTextInputFormat should include default 
       separator and how to change it (Akira AJISAKA via aw)
 
+    MAPREDUCE-5906. Inconsistent configuration in property 
+      "mapreduce.reduce.shuffle.input.buffer.percent" (Akira AJISAKA via aw)
+
   BUG FIXES
 
     MAPREDUCE-5714. Removed forceful JVM exit in shutDownJob.  
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 4795af78d2a..0a586967d01 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -265,6 +265,7 @@ public interface MRJobConfig {
   public static final String REDUCE_MEMORY_TOTAL_BYTES = "mapreduce.reduce.memory.totalbytes";
 
   public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent";
+  public static final float DEFAULT_SHUFFLE_INPUT_BUFFER_PERCENT = 0.70f;
 
   public static final String SHUFFLE_MEMORY_LIMIT_PERCENT
     = "mapreduce.reduce.shuffle.memory.limit.percent";
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
index 4d83995dcfd..49aa8576d60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -156,7 +156,8 @@ public class MergeManagerImpl implements MergeManager {
     this.rfs = ((LocalFileSystem)localFS).getRaw();
     
     final float maxInMemCopyUse =
-      jobConf.getFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 0.90f);
+      jobConf.getFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT,
+          MRJobConfig.DEFAULT_SHUFFLE_INPUT_BUFFER_PERCENT);
     if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) {
       throw new IllegalArgumentException("Invalid value for " +
           MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT + ": " +

From 399e428deb6a4aafff20bfff6a2fd0acfd21366a Mon Sep 17 00:00:00 2001
From: Allen Wittenauer 
Date: Thu, 14 Aug 2014 20:28:36 +0000
Subject: [PATCH 37/67] HADOOP-10121. Fix javadoc spelling for
 HadoopArchives#writeTopLevelDirs (Akira AJISAKA via aw)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618044 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-common-project/hadoop-common/CHANGES.txt                | 3 +++
 .../src/main/java/org/apache/hadoop/tools/HadoopArchives.java  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9215927bb82..446934f55d9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -430,6 +430,9 @@ Trunk (Unreleased)
     HADOOP-10862. Miscellaneous trivial corrections to KMS classes. 
     (asuresh via tucu)
 
+    HADOOP-10121. Fix javadoc spelling for HadoopArchives#writeTopLevelDirs
+      (Akira AJISAKA via aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)
diff --git a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
index 98b3c9ca1e9..93994b817a4 100644
--- a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
@@ -338,7 +338,7 @@ public class HadoopArchives implements Tool {
    * directories to
    * @param paths the source paths provided by the user. They
    * are glob free and have full path (not relative paths)
-   * @param parentPath the parent path that you wnat the archives
+   * @param parentPath the parent path that you want the archives
    * to be relative to. example - /home/user/dir1 can be archived with
    * parent as /home or /home/user.
    * @throws IOException

From a9023c2736fb8a95dbba90d3f61c9eca33126af4 Mon Sep 17 00:00:00 2001
From: Zhijie Shen 
Date: Thu, 14 Aug 2014 21:17:20 +0000
Subject: [PATCH 38/67] YARN-2397. Avoided loading two authentication filters
 for RM and TS web interfaces. Contributed by Varun Vasudev.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618054 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-yarn-project/CHANGES.txt               |  3 +
 .../ApplicationHistoryServer.java             | 33 ++++++--
 .../TestApplicationHistoryServer.java         | 28 +++++++
 .../RMAuthenticationFilterInitializer.java    |  2 +-
 .../resourcemanager/ResourceManager.java      | 78 +++++++++++++------
 .../resourcemanager/TestResourceManager.java  | 76 ++++++++++++++++++
 ...ServicesDelegationTokenAuthentication.java |  3 +
 7 files changed, 190 insertions(+), 33 deletions(-)

diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4b06e627fca..84216274eb5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -193,6 +193,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2070. Made DistributedShell publish the short user name to the timeline
     server. (Robert Kanter via zjshen)
 
+    YARN-2397. Avoided loading two authentication filters for RM and TS web
+    interfaces. (Varun Vasudev via zjshen)
+
 Release 2.5.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 29bbd217338..c61b80e1993 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -27,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.service.Service;
@@ -195,14 +197,31 @@ public class ApplicationHistoryServer extends CompositeService {
     // the customized filter will be loaded by the timeline server to do Kerberos
     // + DT authentication.
     String initializers = conf.get("hadoop.http.filter.initializers");
+
     initializers =
-        initializers == null || initializers.length() == 0 ? "" : ","
-            + initializers;
-    if (!initializers.contains(
-        TimelineAuthenticationFilterInitializer.class.getName())) {
-      conf.set("hadoop.http.filter.initializers",
-          TimelineAuthenticationFilterInitializer.class.getName()
-              + initializers);
+        initializers == null || initializers.length() == 0 ? "" : initializers;
+
+    if (!initializers.contains(TimelineAuthenticationFilterInitializer.class
+      .getName())) {
+      initializers =
+          TimelineAuthenticationFilterInitializer.class.getName() + ","
+              + initializers;
+    }
+
+    String[] parts = initializers.split(",");
+    ArrayList target = new ArrayList();
+    for (String filterInitializer : parts) {
+      filterInitializer = filterInitializer.trim();
+      if (filterInitializer.equals(AuthenticationFilterInitializer.class
+        .getName())) {
+        continue;
+      }
+      target.add(filterInitializer);
+    }
+    String actualInitializers =
+        org.apache.commons.lang.StringUtils.join(target, ",");
+    if (!actualInitializers.equals(initializers)) {
+      conf.set("hadoop.http.filter.initializers", actualInitializers);
     }
     String bindAddress = WebAppUtils.getWebAppBindURL(conf,
                           YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
index 5c55becb6c9..bcd8e454c5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
@@ -23,11 +23,14 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.fail;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestApplicationHistoryServer {
@@ -69,6 +72,31 @@ public class TestApplicationHistoryServer {
     }
   }
 
+  @Test(timeout = 50000)
+  public void testFilteOverrides() throws Exception {
+
+    String[] filterInitializers =
+        {
+            AuthenticationFilterInitializer.class.getName(),
+            TimelineAuthenticationFilterInitializer.class.getName(),
+            AuthenticationFilterInitializer.class.getName() + ","
+                + TimelineAuthenticationFilterInitializer.class.getName(),
+            AuthenticationFilterInitializer.class.getName() + ", "
+                + TimelineAuthenticationFilterInitializer.class.getName() };
+    for (String filterInitializer : filterInitializers) {
+      historyServer = new ApplicationHistoryServer();
+      Configuration config = new YarnConfiguration();
+      config.set("hadoop.http.filter.initializers", filterInitializer);
+      historyServer.init(config);
+      historyServer.start();
+      Configuration tmp = historyServer.getConfig();
+      assertEquals(TimelineAuthenticationFilterInitializer.class.getName(),
+        tmp.get("hadoop.http.filter.initializers"));
+      historyServer.stop();
+      AHSWebApp.resetInstance();
+    }
+  }
+
   @After
   public void stop() {
     if (historyServer != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java
index 2227833e7cf..128794ee6eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java
@@ -114,7 +114,7 @@ public class RMAuthenticationFilterInitializer extends FilterInitializer {
   public void initFilter(FilterContainer container, Configuration conf) {
 
     Map filterConfig = createFilterConfig(conf);
-    container.addFilter("YARNAuthenticationFilter",
+    container.addFilter("RMAuthenticationFilter",
       RMAuthenticationFilter.class.getName(), filterConfig);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 90441d6c947..6fea90043bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -35,6 +36,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.http.lib.StaticUserWebFilter;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.security.Groups;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -799,10 +801,11 @@ public class ResourceManager extends CompositeService implements Recoverable {
 
     // Use the customized yarn filter instead of the standard kerberos filter to
     // allow users to authenticate using delegation tokens
-    // 3 conditions need to be satisfied -
+    // 4 conditions need to be satisfied -
     // 1. security is enabled
     // 2. http auth type is set to kerberos
     // 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true
+    // 4. hadoop.http.filter.initializers container AuthenticationFilterInitializer
 
     Configuration conf = getConfig();
     boolean useYarnAuthenticationFilter =
@@ -811,41 +814,66 @@ public class ResourceManager extends CompositeService implements Recoverable {
           YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER);
     String authPrefix = "hadoop.http.authentication.";
     String authTypeKey = authPrefix + "type";
-    String initializers = conf.get("hadoop.http.filter.initializers");
-    if (UserGroupInformation.isSecurityEnabled()
-        && useYarnAuthenticationFilter
-        && conf.get(authTypeKey, "").equalsIgnoreCase(
-          KerberosAuthenticationHandler.TYPE)) {
-      LOG.info("Using RM authentication filter(kerberos/delegation-token)"
-          + " for RM webapp authentication");
-      RMAuthenticationHandler
-        .setSecretManager(getClientRMService().rmDTSecretManager);
-      String yarnAuthKey =
-          authPrefix + RMAuthenticationFilter.AUTH_HANDLER_PROPERTY;
-      conf.setStrings(yarnAuthKey, RMAuthenticationHandler.class.getName());
+    String filterInitializerConfKey = "hadoop.http.filter.initializers";
+    String actualInitializers = "";
+    Class[] initializersClasses =
+        conf.getClasses(filterInitializerConfKey);
 
-      initializers =
-          initializers == null || initializers.isEmpty() ? "" : ","
-              + initializers;
-      if (!initializers.contains(RMAuthenticationFilterInitializer.class
-        .getName())) {
-        conf.set("hadoop.http.filter.initializers",
-          RMAuthenticationFilterInitializer.class.getName() + initializers);
+    boolean hasHadoopAuthFilterInitializer = false;
+    boolean hasRMAuthFilterInitializer = false;
+    if (initializersClasses != null) {
+      for (Class initializer : initializersClasses) {
+        if (initializer.getName().equals(
+          AuthenticationFilterInitializer.class.getName())) {
+          hasHadoopAuthFilterInitializer = true;
+        }
+        if (initializer.getName().equals(
+          RMAuthenticationFilterInitializer.class.getName())) {
+          hasRMAuthFilterInitializer = true;
+        }
+      }
+      if (UserGroupInformation.isSecurityEnabled()
+          && useYarnAuthenticationFilter
+          && hasHadoopAuthFilterInitializer
+          && conf.get(authTypeKey, "").equals(
+            KerberosAuthenticationHandler.TYPE)) {
+        ArrayList target = new ArrayList();
+        for (Class filterInitializer : initializersClasses) {
+          if (filterInitializer.getName().equals(
+            AuthenticationFilterInitializer.class.getName())) {
+            if (hasRMAuthFilterInitializer == false) {
+              target.add(RMAuthenticationFilterInitializer.class.getName());
+            }
+            continue;
+          }
+          target.add(filterInitializer.getName());
+        }
+        actualInitializers = StringUtils.join(",", target);
+
+        LOG.info("Using RM authentication filter(kerberos/delegation-token)"
+            + " for RM webapp authentication");
+        RMAuthenticationHandler
+          .setSecretManager(getClientRMService().rmDTSecretManager);
+        String yarnAuthKey =
+            authPrefix + RMAuthenticationFilter.AUTH_HANDLER_PROPERTY;
+        conf.setStrings(yarnAuthKey, RMAuthenticationHandler.class.getName());
+        conf.set(filterInitializerConfKey, actualInitializers);
       }
     }
 
-    // if security is not enabled and the default filter initializer has been
-    // set, set the initializer to include the
+    // if security is not enabled and the default filter initializer has not 
+    // been set, set the initializer to include the
     // RMAuthenticationFilterInitializer which in turn will set up the simple
     // auth filter.
 
+    String initializers = conf.get(filterInitializerConfKey);
     if (!UserGroupInformation.isSecurityEnabled()) {
-      if (initializers == null || initializers.isEmpty()) {
-        conf.set("hadoop.http.filter.initializers",
+      if (initializersClasses == null || initializersClasses.length == 0) {
+        conf.set(filterInitializerConfKey,
           RMAuthenticationFilterInitializer.class.getName());
         conf.set(authTypeKey, "simple");
       } else if (initializers.equals(StaticUserWebFilter.class.getName())) {
-        conf.set("hadoop.http.filter.initializers",
+        conf.set(filterInitializerConfKey,
           RMAuthenticationFilterInitializer.class.getName() + ","
               + initializers);
         conf.set(authTypeKey, "simple");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
index 4fdfcf61e94..1117fbe3393 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
@@ -27,7 +27,10 @@ import java.util.Collection;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
 import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -39,8 +42,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -235,4 +240,75 @@ public class TestResourceManager {
     }
   }
 
+  @Test(timeout = 50000)
+  public void testFilterOverrides() throws Exception {
+    String filterInitializerConfKey = "hadoop.http.filter.initializers";
+    String[] filterInitializers =
+        {
+            AuthenticationFilterInitializer.class.getName(),
+            RMAuthenticationFilterInitializer.class.getName(),
+            AuthenticationFilterInitializer.class.getName() + ","
+                + RMAuthenticationFilterInitializer.class.getName(),
+            AuthenticationFilterInitializer.class.getName() + ", "
+                + RMAuthenticationFilterInitializer.class.getName(),
+            AuthenticationFilterInitializer.class.getName() + ", "
+                + this.getClass().getName() };
+    for (String filterInitializer : filterInitializers) {
+      resourceManager = new ResourceManager();
+      Configuration conf = new YarnConfiguration();
+      conf.set(filterInitializerConfKey, filterInitializer);
+      conf.set("hadoop.security.authentication", "kerberos");
+      conf.set("hadoop.http.authentication.type", "kerberos");
+      try {
+        try {
+          UserGroupInformation.setConfiguration(conf);
+        } catch (Exception e) {
+          // ignore we just care about getting true for
+          // isSecurityEnabled()
+          LOG.info("Got expected exception");
+        }
+        resourceManager.init(conf);
+        resourceManager.startWepApp();
+      } catch (RuntimeException e) {
+        // Exceptions are expected because we didn't setup everything
+        // just want to test filter settings
+        String tmp = resourceManager.getConfig().get(filterInitializerConfKey);
+        if (filterInitializer.contains(this.getClass().getName())) {
+          Assert.assertEquals(RMAuthenticationFilterInitializer.class.getName()
+              + "," + this.getClass().getName(), tmp);
+        } else {
+          Assert.assertEquals(
+            RMAuthenticationFilterInitializer.class.getName(), tmp);
+        }
+        resourceManager.stop();
+      }
+    }
+
+    // simple mode overrides
+    String[] simpleFilterInitializers =
+        { "", StaticUserWebFilter.class.getName() };
+    for (String filterInitializer : simpleFilterInitializers) {
+      resourceManager = new ResourceManager();
+      Configuration conf = new YarnConfiguration();
+      conf.set(filterInitializerConfKey, filterInitializer);
+      try {
+        UserGroupInformation.setConfiguration(conf);
+        resourceManager.init(conf);
+        resourceManager.startWepApp();
+      } catch (RuntimeException e) {
+        // Exceptions are expected because we didn't setup everything
+        // just want to test filter settings
+        String tmp = resourceManager.getConfig().get(filterInitializerConfKey);
+        if (filterInitializer.equals(StaticUserWebFilter.class.getName())) {
+          Assert.assertEquals(RMAuthenticationFilterInitializer.class.getName()
+              + "," + StaticUserWebFilter.class.getName(), tmp);
+        } else {
+          Assert.assertEquals(
+            RMAuthenticationFilterInitializer.class.getName(), tmp);
+        }
+        resourceManager.stop();
+      }
+    }
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
index 34a914a7541..239d5922eaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
@@ -41,6 +41,7 @@ import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.KerberosTestUtils;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
@@ -122,6 +123,8 @@ public class TestRMWebServicesDelegationTokenAuthentication {
       "kerberos");
     rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
       true);
+    rmconf.set("hadoop.http.filter.initializers",
+      AuthenticationFilterInitializer.class.getName());
     rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
       httpSpnegoPrincipal);
     rmconf.set(YarnConfiguration.RM_KEYTAB,

From 597b3fdc925f9eab69be1a72eee135216251e919 Mon Sep 17 00:00:00 2001
From: Allen Wittenauer 
Date: Thu, 14 Aug 2014 21:39:49 +0000
Subject: [PATCH 39/67] HADOOP-10231. Add some components in Native Libraries
 document (Akira AJISAKA via aw)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618059 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-common-project/hadoop-common/CHANGES.txt    |  3 +++
 .../src/site/apt/NativeLibraries.apt.vm            | 14 +++++++++-----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 446934f55d9..38a21a4a847 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -205,6 +205,9 @@ Trunk (Unreleased)
     HADOOP-8944. Shell command fs -count should include human readable option
       (Jonathan Allen via aw)
 
+    HADOOP-10231. Add some components in Native Libraries document (Akira 
+      AJISAKA via aw)
+
   BUG FIXES
 
     HADOOP-9451. Fault single-layer config if node group topology is enabled.
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm
index 27325194100..070a7d3e568 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm
@@ -30,6 +30,8 @@ Native Libraries Guide
    compression" could refer to all *.so's you need to compile that are
    specifically related to compression. Currently, however, this document
    only addresses the native hadoop library (<<>>).
+   The document for libhdfs library (<<>>) is
+   {{{../hadoop-hdfs/LibHdfs.html}here}}.
 
 * Native Hadoop Library
 
@@ -64,14 +66,16 @@ Native Libraries Guide
 
 * Components
 
-   The native hadoop library includes two components, the zlib and gzip
-   compression codecs:
+   The native hadoop library includes various components:
 
-     * zlib
+   * Compression Codecs (bzip2, lz4, snappy, zlib)
 
-     * gzip
+   * Native IO utilities for {{{../hadoop-hdfs/ShortCircuitLocalReads.html}
+     HDFS Short-Circuit Local Reads}} and
+     {{{../hadoop-hdfs/CentralizedCacheManagement.html}Centralized Cache
+     Management in HDFS}}
 
-   The native hadoop library is imperative for gzip to work.
+   * CRC32 checksum implementation
 
 * Supported Platforms
 

From b8ea45c2e33e0365ee3c0308da4681dc9cb4a285 Mon Sep 17 00:00:00 2001
From: Allen Wittenauer 
Date: Thu, 14 Aug 2014 21:45:58 +0000
Subject: [PATCH 40/67] YARN-2197. Add a link to YARN CHANGES.txt in the left
 side of doc (Akira AJISAKA via aw)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618066 13f79535-47bb-0310-9956-ffa450edef68
---
 hadoop-project/src/site/site.xml | 1 +
 hadoop-yarn-project/CHANGES.txt  | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 37e0153ec8f..3930ace2802 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -142,6 +142,7 @@
       
       
       
+
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 84216274eb5..920983dd7c6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -8,6 +8,9 @@ Trunk - Unreleased IMPROVEMENTS + YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc + (Akira AJISAKA via aw) + OPTIMIZATIONS BUG FIXES From 0350ea3c72b8cb036f8f066c12f400dd9e45c439 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 14 Aug 2014 21:53:12 +0000 Subject: [PATCH 41/67] YARN-1918. Typo in description and error message for yarn.resourcemanager.cluster-id (Anandha L Ranganathan via aw) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618070 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 920983dd7c6..b879600537b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -11,6 +11,9 @@ Trunk - Unreleased YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc (Akira AJISAKA via aw) + YARN-1918. Typo in description and error message for + 'yarn.resourcemanager.cluster-id' (Anandha L Ranganathan via aw) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9e08ef52008..39d1dd3b015 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1370,7 +1370,7 @@ public class YarnConfiguration extends Configuration { public static String getClusterId(Configuration conf) { String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID); if (clusterId == null) { - throw new HadoopIllegalArgumentException("Configuration doesn't specify" + + throw new HadoopIllegalArgumentException("Configuration doesn't specify " + YarnConfiguration.RM_CLUSTER_ID); } return clusterId; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 94b4d7f20ec..9b2b6764923 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -474,7 +474,7 @@ Name of the cluster. In a HA setting, this is used to ensure the RM participates in leader - election fo this cluster and ensures it does not affect + election for this cluster and ensures it does not affect other clusters yarn.resourcemanager.cluster-id From 7378af7bcad781353ca40bd9754bf958480381a1 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Thu, 14 Aug 2014 23:17:33 +0000 Subject: [PATCH 42/67] MAPREDUCE-5878. some standard JDK APIs are not part of system classes defaults. Contributed by Sangjin Lee git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618080 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/mapred-default.xml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 7dc4436a887..05cce4ebde8 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -221,6 +221,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6010. HistoryServerFileSystemStateStore fails to update tokens (jlowe) + MAPREDUCE-5878. some standard JDK APIs are not part of system classes + defaults (Sangjin Lee via jlowe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 8e7e76c3442..110e3162833 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1225,9 +1225,9 @@ mapreduce.job.classloader.system.classes - java.,javax.,org.apache.commons.logging.,org.apache.log4j., - org.apache.hadoop.,core-default.xml,hdfs-default.xml, - mapred-default.xml,yarn-default.xml + java.,javax.,org.w3c.dom.,org.xml.sax.,org.apache.commons.logging., + org.apache.log4j.,org.apache.hadoop.,core-default.xml, + hdfs-default.xml,mapred-default.xml,yarn-default.xml A comma-separated list of classes that should be loaded from the system classpath, not the user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending in '.' (period) are treated as package names, From 894b4e427fdcce08ebcf6b6fc04bf8d8e573f64f Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 15 Aug 2014 02:33:24 +0000 Subject: [PATCH 43/67] HDFS-6850. Move NFS out of order write unit tests into TestWrites class. Contributed by Zhe Zhang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618091 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/hdfs/nfs/nfs3/TestWrites.java | 77 +++++++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + 2 files changed, 80 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java index 32f9b2bcddf..363bc1205e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java @@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentNavigableMap; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys; import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration; import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.COMMIT_STATUS; import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx; @@ -407,4 +408,80 @@ public class TestWrites { } } } + + @Test + public void testOOOWrites() throws IOException, InterruptedException { + NfsConfiguration config = new NfsConfiguration(); + MiniDFSCluster cluster = null; + RpcProgramNfs3 nfsd; + final int bufSize = 32; + final int numOOO = 3; + SecurityHandler securityHandler = Mockito.mock(SecurityHandler.class); + Mockito.when(securityHandler.getUser()).thenReturn( + System.getProperty("user.name")); + String currentUser = System.getProperty("user.name"); + config.set( + DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserGroupConfKey(currentUser), + "*"); + config.set( + DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserIpConfKey(currentUser), + "*"); + ProxyUsers.refreshSuperUserGroupsConfiguration(config); + // Use emphral port in case tests are running in parallel + config.setInt("nfs3.mountd.port", 0); + config.setInt("nfs3.server.port", 0); + + try { + cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build(); + cluster.waitActive(); + + Nfs3 nfs3 = new Nfs3(config); + nfs3.startServiceInternal(false); + nfsd = (RpcProgramNfs3) nfs3.getRpcProgram(); + + DFSClient dfsClient = new DFSClient(NameNode.getAddress(config), config); + HdfsFileStatus status = dfsClient.getFileInfo("/"); + FileHandle rootHandle = new FileHandle(status.getFileId()); + + CREATE3Request createReq = new CREATE3Request(rootHandle, + "out-of-order-write" + System.currentTimeMillis(), + Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0); + XDR createXdr = new XDR(); + createReq.serialize(createXdr); + CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", 1234)); + FileHandle handle = createRsp.getObjHandle(); + + byte[][] oooBuf = new byte[numOOO][bufSize]; + for (int i = 0; i < numOOO; i++) { + Arrays.fill(oooBuf[i], (byte) i); + } + + for (int i = 0; i < numOOO; i++) { + final long offset = (numOOO - 1 - i) * bufSize; + WRITE3Request writeReq = new WRITE3Request(handle, offset, bufSize, + WriteStableHow.UNSTABLE, ByteBuffer.wrap(oooBuf[i])); + XDR writeXdr = new XDR(); + writeReq.serialize(writeXdr); + nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler, + new InetSocketAddress("localhost", 1234)); + } + + waitWrite(nfsd, handle, 60000); + READ3Request readReq = new READ3Request(handle, bufSize, bufSize); + XDR readXdr = new XDR(); + readReq.serialize(readXdr); + READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(), + securityHandler, new InetSocketAddress("localhost", config.getInt( + NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, + NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT))); + assertTrue(Arrays.equals(oooBuf[1], readRsp.getData().array())); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 393938ce4ef..b47018a61b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -402,6 +402,9 @@ Release 2.6.0 - UNRELEASED HDFS-6849. Replace HttpFS custom proxyuser handling with common implementation. (tucu) + HDFS-6850. Move NFS out of order write unit tests into TestWrites class. + (Zhe Zhang via atm) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) From e86c9ef6517313aaa0e4575261a462f2f55d43dc Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 15 Aug 2014 05:03:58 +0000 Subject: [PATCH 44/67] HADOOP-10770. KMS add delegation token support. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618096 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../KeyProviderDelegationTokenExtension.java | 8 +- .../crypto/key/kms/KMSClientProvider.java | 54 +++- .../DelegationTokenAuthenticationHandler.java | 2 +- .../hadoop/crypto/key/kms/server/KMS.java | 272 ++++++++++++------ .../hadoop/crypto/key/kms/server/KMSACLs.java | 3 +- .../crypto/key/kms/server/KMSAudit.java | 18 +- .../kms/server/KMSAuthenticationFilter.java | 21 +- .../key/kms/server/KMSExceptionsProvider.java | 8 +- .../crypto/key/kms/server/KMSMDCFilter.java | 17 +- .../hadoop-kms/src/site/apt/index.apt.vm | 40 +++ .../hadoop/crypto/key/kms/server/TestKMS.java | 98 ++++++- .../crypto/key/kms/server/TestKMSACLs.java | 10 +- .../crypto/key/kms/server/TestKMSAudit.java | 10 +- 14 files changed, 418 insertions(+), 145 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 38a21a4a847..0a17758483c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -208,6 +208,8 @@ Trunk (Unreleased) HADOOP-10231. Add some components in Native Libraries document (Akira AJISAKA via aw) + HADOOP-10770. KMS add delegation token support. (tucu) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java index ccf382241ff..2f237c63968 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java @@ -20,6 +20,8 @@ package org.apache.hadoop.crypto.key; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; +import java.io.IOException; + /** * A KeyProvider extension with the ability to add a renewer's Delegation * Tokens to the provided Credentials. @@ -45,9 +47,10 @@ public class KeyProviderDelegationTokenExtension extends * @param renewer the user allowed to renew the delegation tokens * @param credentials cache in which to add new delegation tokens * @return list of new delegation tokens + * @throws IOException thrown if IOException if an IO error occurs. */ public Token[] addDelegationTokens(final String renewer, - Credentials credentials); + Credentials credentials) throws IOException; } /** @@ -76,9 +79,10 @@ public class KeyProviderDelegationTokenExtension extends * @param renewer the user allowed to renew the delegation tokens * @param credentials cache in which to add new delegation tokens * @return list of new delegation tokens + * @throws IOException thrown if IOException if an IO error occurs. */ public Token[] addDelegationTokens(final String renewer, - Credentials credentials) { + Credentials credentials) throws IOException { return getExtension().addDelegationTokens(renewer, credentials); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index cf5b11315fa..35439f26f6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -22,15 +22,17 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; +import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.ProviderUtils; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; -import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; import org.apache.hadoop.security.ssl.SSLFactory; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.http.client.utils.URIBuilder; import org.codehaus.jackson.map.ObjectMapper; @@ -69,7 +71,10 @@ import com.google.common.base.Preconditions; * KMS client KeyProvider implementation. */ @InterfaceAudience.Private -public class KMSClientProvider extends KeyProvider implements CryptoExtension { +public class KMSClientProvider extends KeyProvider implements CryptoExtension, + KeyProviderDelegationTokenExtension.DelegationTokenExtension { + + public static final String TOKEN_KIND = "kms-dt"; public static final String SCHEME_NAME = "kms"; @@ -229,6 +234,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension { private String kmsUrl; private SSLFactory sslFactory; private ConnectionConfigurator configurator; + private DelegationTokenAuthenticatedURL.Token authToken; @Override public String toString() { @@ -309,6 +315,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension { CommonConfigurationKeysPublic. KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT), new EncryptedQueueRefiller()); + authToken = new DelegationTokenAuthenticatedURL.Token(); } private String createServiceURL(URL url) throws IOException { @@ -325,12 +332,14 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension { try { StringBuilder sb = new StringBuilder(); sb.append(kmsUrl); - sb.append(collection); - if (resource != null) { - sb.append("/").append(URLEncoder.encode(resource, UTF8)); - } - if (subResource != null) { - sb.append("/").append(subResource); + if (collection != null) { + sb.append(collection); + if (resource != null) { + sb.append("/").append(URLEncoder.encode(resource, UTF8)); + if (subResource != null) { + sb.append("/").append(subResource); + } + } } URIBuilder uriBuilder = new URIBuilder(sb.toString()); if (parameters != null) { @@ -369,9 +378,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension { throws IOException { HttpURLConnection conn; try { - AuthenticatedURL authUrl = new AuthenticatedURL(new PseudoAuthenticator(), - configurator); - conn = authUrl.openConnection(url, new AuthenticatedURL.Token()); + DelegationTokenAuthenticatedURL authUrl = + new DelegationTokenAuthenticatedURL(configurator); + conn = authUrl.openConnection(url, authToken); } catch (AuthenticationException ex) { throw new IOException(ex); } @@ -729,4 +738,25 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension { } } + @Override + public Token[] addDelegationTokens(String renewer, + Credentials credentials) throws IOException { + Token[] tokens; + URL url = createURL(null, null, null, null); + DelegationTokenAuthenticatedURL authUrl = + new DelegationTokenAuthenticatedURL(configurator); + try { + Token token = authUrl.getDelegationToken(url, authToken, renewer); + if (token != null) { + credentials.addToken(token.getService(), token); + tokens = new Token[] { token }; + } else { + throw new IOException("Got NULL as delegation token"); + } + } catch (AuthenticationException ex) { + throw new IOException(ex); + } + return tokens; + } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java index 3b6c289df5f..670ec551a09 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java @@ -75,7 +75,7 @@ public abstract class DelegationTokenAuthenticationHandler public static final String PREFIX = "delegation-token."; - public static final String TOKEN_KIND = PREFIX + "token-kind.sec"; + public static final String TOKEN_KIND = PREFIX + "token-kind"; public static final String UPDATE_INTERVAL = PREFIX + "update-interval.sec"; public static final long UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60; diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java index 08cb91f6504..faec70a7554 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -25,9 +25,10 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.crypto.key.kms.KMSRESTConstants; import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -38,15 +39,13 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import javax.ws.rs.core.SecurityContext; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.security.Principal; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -74,15 +73,6 @@ public class KMS { kmsAudit= KMSWebApp.getKMSAudit(); } - private static Principal getPrincipal(SecurityContext securityContext) - throws AuthenticationException{ - Principal user = securityContext.getUserPrincipal(); - if (user == null) { - throw new AuthenticationException("User must be authenticated"); - } - return user; - } - private static final String UNAUTHORIZED_MSG_WITH_KEY = "User:%s not allowed to do '%s' on '%s'"; @@ -90,20 +80,21 @@ public class KMS { private static final String UNAUTHORIZED_MSG_WITHOUT_KEY = "User:%s not allowed to do '%s'"; - private void assertAccess(KMSACLs.Type aclType, Principal principal, + private void assertAccess(KMSACLs.Type aclType, UserGroupInformation ugi, KMSOp operation) throws AccessControlException { - assertAccess(aclType, principal, operation, null); + assertAccess(aclType, ugi, operation, null); } - private void assertAccess(KMSACLs.Type aclType, Principal principal, - KMSOp operation, String key) throws AccessControlException { - if (!KMSWebApp.getACLs().hasAccess(aclType, principal.getName())) { + private void assertAccess(KMSACLs.Type aclType, + UserGroupInformation ugi, KMSOp operation, String key) + throws AccessControlException { + if (!KMSWebApp.getACLs().hasAccess(aclType, ugi)) { KMSWebApp.getUnauthorizedCallsMeter().mark(); - kmsAudit.unauthorized(principal, operation, key); + kmsAudit.unauthorized(ugi, operation, key); throw new AuthorizationException(String.format( (key != null) ? UNAUTHORIZED_MSG_WITH_KEY : UNAUTHORIZED_MSG_WITHOUT_KEY, - principal.getName(), operation, key)); + ugi.getShortUserName(), operation, key)); } } @@ -123,15 +114,14 @@ public class KMS { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @SuppressWarnings("unchecked") - public Response createKey(@Context SecurityContext securityContext, - Map jsonKey) throws Exception { + public Response createKey(Map jsonKey) throws Exception { KMSWebApp.getAdminCallsMeter().mark(); - Principal user = getPrincipal(securityContext); - String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD); + UserGroupInformation user = HttpUserGroupInformation.get(); + final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD); KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD); assertAccess(KMSACLs.Type.CREATE, user, KMSOp.CREATE_KEY, name); String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD); - String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD); + final String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD); int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD)) ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0; String description = (String) @@ -142,7 +132,7 @@ public class KMS { assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, KMSOp.CREATE_KEY, name); } - KeyProvider.Options options = new KeyProvider.Options( + final KeyProvider.Options options = new KeyProvider.Options( KMSWebApp.getConfiguration()); if (cipher != null) { options.setCipher(cipher); @@ -153,16 +143,23 @@ public class KMS { options.setDescription(description); options.setAttributes(attributes); - KeyProvider.KeyVersion keyVersion = (material != null) - ? provider.createKey(name, Base64.decodeBase64(material), options) - : provider.createKey(name, options); - - provider.flush(); + KeyProvider.KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + KeyProvider.KeyVersion keyVersion = (material != null) + ? provider.createKey(name, Base64.decodeBase64(material), options) + : provider.createKey(name, options); + provider.flush(); + return keyVersion; + } + } + ); kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" + (material != null) + " Description:" + description); - if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user.getName())) { + if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) { keyVersion = removeKeyMaterial(keyVersion); } Map json = KMSServerJSONUtils.toJSON(keyVersion); @@ -176,14 +173,21 @@ public class KMS { @DELETE @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") - public Response deleteKey(@Context SecurityContext securityContext, - @PathParam("name") String name) throws Exception { + public Response deleteKey(@PathParam("name") final String name) + throws Exception { KMSWebApp.getAdminCallsMeter().mark(); - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); assertAccess(KMSACLs.Type.DELETE, user, KMSOp.DELETE_KEY, name); KMSClientProvider.checkNotEmpty(name, "name"); - provider.deleteKey(name); - provider.flush(); + + user.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + provider.deleteKey(name); + provider.flush(); + return null; + } + }); kmsAudit.ok(user, KMSOp.DELETE_KEY, name, ""); @@ -194,29 +198,36 @@ public class KMS { @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - public Response rolloverKey(@Context SecurityContext securityContext, - @PathParam("name") String name, Map jsonMaterial) - throws Exception { + public Response rolloverKey(@PathParam("name") final String name, + Map jsonMaterial) throws Exception { KMSWebApp.getAdminCallsMeter().mark(); - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); assertAccess(KMSACLs.Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name); KMSClientProvider.checkNotEmpty(name, "name"); - String material = (String) + final String material = (String) jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD); if (material != null) { assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, KMSOp.ROLL_NEW_VERSION, name); } - KeyProvider.KeyVersion keyVersion = (material != null) - ? provider.rollNewVersion(name, Base64.decodeBase64(material)) - : provider.rollNewVersion(name); - provider.flush(); + KeyProvider.KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + KeyVersion keyVersion = (material != null) + ? provider.rollNewVersion(name, Base64.decodeBase64(material)) + : provider.rollNewVersion(name); + provider.flush(); + return keyVersion; + } + } + ); kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" + (material != null) + " NewVersion:" + keyVersion.getVersionName()); - if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user.getName())) { + if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) { keyVersion = removeKeyMaterial(keyVersion); } Map json = KMSServerJSONUtils.toJSON(keyVersion); @@ -226,14 +237,23 @@ public class KMS { @GET @Path(KMSRESTConstants.KEYS_METADATA_RESOURCE) @Produces(MediaType.APPLICATION_JSON) - public Response getKeysMetadata(@Context SecurityContext securityContext, - @QueryParam(KMSRESTConstants.KEY) List keyNamesList) - throws Exception { + public Response getKeysMetadata(@QueryParam(KMSRESTConstants.KEY) + List keyNamesList) throws Exception { KMSWebApp.getAdminCallsMeter().mark(); - Principal user = getPrincipal(securityContext); - String[] keyNames = keyNamesList.toArray(new String[keyNamesList.size()]); + UserGroupInformation user = HttpUserGroupInformation.get(); + final String[] keyNames = keyNamesList.toArray( + new String[keyNamesList.size()]); assertAccess(KMSACLs.Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA); - KeyProvider.Metadata[] keysMeta = provider.getKeysMetadata(keyNames); + + KeyProvider.Metadata[] keysMeta = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyProvider.Metadata[] run() throws Exception { + return provider.getKeysMetadata(keyNames); + } + } + ); + Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta); kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, ""); return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); @@ -242,36 +262,52 @@ public class KMS { @GET @Path(KMSRESTConstants.KEYS_NAMES_RESOURCE) @Produces(MediaType.APPLICATION_JSON) - public Response getKeyNames(@Context SecurityContext securityContext) - throws Exception { + public Response getKeyNames() throws Exception { KMSWebApp.getAdminCallsMeter().mark(); - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); assertAccess(KMSACLs.Type.GET_KEYS, user, KMSOp.GET_KEYS); - Object json = provider.getKeys(); + + List json = user.doAs( + new PrivilegedExceptionAction>() { + @Override + public List run() throws Exception { + return provider.getKeys(); + } + } + ); + kmsAudit.ok(user, KMSOp.GET_KEYS, ""); return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); } @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") - public Response getKey(@Context SecurityContext securityContext, - @PathParam("name") String name) + public Response getKey(@PathParam("name") String name) throws Exception { - return getMetadata(securityContext, name); + return getMetadata(name); } @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.METADATA_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON) - public Response getMetadata(@Context SecurityContext securityContext, - @PathParam("name") String name) + public Response getMetadata(@PathParam("name") final String name) throws Exception { - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); KMSClientProvider.checkNotEmpty(name, "name"); KMSWebApp.getAdminCallsMeter().mark(); assertAccess(KMSACLs.Type.GET_METADATA, user, KMSOp.GET_METADATA, name); - Object json = KMSServerJSONUtils.toJSON(name, provider.getMetadata(name)); + + KeyProvider.Metadata metadata = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyProvider.Metadata run() throws Exception { + return provider.getMetadata(name); + } + } + ); + + Object json = KMSServerJSONUtils.toJSON(name, metadata); kmsAudit.ok(user, KMSOp.GET_METADATA, name, ""); return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); } @@ -280,14 +316,23 @@ public class KMS { @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.CURRENT_VERSION_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON) - public Response getCurrentVersion(@Context SecurityContext securityContext, - @PathParam("name") String name) + public Response getCurrentVersion(@PathParam("name") final String name) throws Exception { - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); KMSClientProvider.checkNotEmpty(name, "name"); KMSWebApp.getKeyCallsMeter().mark(); assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_CURRENT_KEY, name); - Object json = KMSServerJSONUtils.toJSON(provider.getCurrentKey(name)); + + KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + return provider.getCurrentKey(name); + } + } + ); + + Object json = KMSServerJSONUtils.toJSON(keyVersion); kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, ""); return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); } @@ -295,14 +340,22 @@ public class KMS { @GET @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}") @Produces(MediaType.APPLICATION_JSON) - public Response getKeyVersion(@Context SecurityContext securityContext, - @PathParam("versionName") String versionName) - throws Exception { - Principal user = getPrincipal(securityContext); + public Response getKeyVersion( + @PathParam("versionName") final String versionName) throws Exception { + UserGroupInformation user = HttpUserGroupInformation.get(); KMSClientProvider.checkNotEmpty(versionName, "versionName"); KMSWebApp.getKeyCallsMeter().mark(); - KeyVersion keyVersion = provider.getKeyVersion(versionName); assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSION); + + KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + return provider.getKeyVersion(versionName); + } + } + ); + if (keyVersion != null) { kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), ""); } @@ -316,13 +369,12 @@ public class KMS { KMSRESTConstants.EEK_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON) public Response generateEncryptedKeys( - @Context SecurityContext securityContext, - @PathParam("name") String name, + @PathParam("name") final String name, @QueryParam(KMSRESTConstants.EEK_OP) String edekOp, @DefaultValue("1") - @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) int numKeys) + @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int numKeys) throws Exception { - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); KMSClientProvider.checkNotEmpty(name, "name"); KMSClientProvider.checkNotNull(edekOp, "eekOp"); @@ -330,12 +382,22 @@ public class KMS { if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) { assertAccess(KMSACLs.Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK, name); - List retEdeks = + final List retEdeks = new LinkedList(); try { - for (int i = 0; i < numKeys; i ++) { - retEdeks.add(provider.generateEncryptedKey(name)); - } + + user.doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + for (int i = 0; i < numKeys; i++) { + retEdeks.add(provider.generateEncryptedKey(name)); + } + return null; + } + } + ); + } catch (Exception e) { throw new IOException(e); } @@ -359,16 +421,17 @@ public class KMS { @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}/" + KMSRESTConstants.EEK_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON) - public Response decryptEncryptedKey(@Context SecurityContext securityContext, - @PathParam("versionName") String versionName, + public Response decryptEncryptedKey( + @PathParam("versionName") final String versionName, @QueryParam(KMSRESTConstants.EEK_OP) String eekOp, Map jsonPayload) throws Exception { - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); KMSClientProvider.checkNotEmpty(versionName, "versionName"); KMSClientProvider.checkNotNull(eekOp, "eekOp"); - String keyName = (String) jsonPayload.get(KMSRESTConstants.NAME_FIELD); + final String keyName = (String) jsonPayload.get( + KMSRESTConstants.NAME_FIELD); String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD); String encMaterialStr = (String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD); @@ -376,14 +439,24 @@ public class KMS { if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) { assertAccess(KMSACLs.Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, keyName); KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD); - byte[] iv = Base64.decodeBase64(ivStr); + final byte[] iv = Base64.decodeBase64(ivStr); KMSClientProvider.checkNotNull(encMaterialStr, KMSRESTConstants.MATERIAL_FIELD); - byte[] encMaterial = Base64.decodeBase64(encMaterialStr); - KeyProvider.KeyVersion retKeyVersion = - provider.decryptEncryptedKey( - new KMSClientProvider.KMSEncryptedKeyVersion(keyName, versionName, - iv, KeyProviderCryptoExtension.EEK, encMaterial)); + final byte[] encMaterial = Base64.decodeBase64(encMaterialStr); + + KeyProvider.KeyVersion retKeyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + return provider.decryptEncryptedKey( + new KMSClientProvider.KMSEncryptedKeyVersion(keyName, + versionName, iv, KeyProviderCryptoExtension.EEK, + encMaterial) + ); + } + } + ); + retJSON = KMSServerJSONUtils.toJSON(retKeyVersion); kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, ""); } else { @@ -400,14 +473,23 @@ public class KMS { @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.VERSIONS_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON) - public Response getKeyVersions(@Context SecurityContext securityContext, - @PathParam("name") String name) + public Response getKeyVersions(@PathParam("name") final String name) throws Exception { - Principal user = getPrincipal(securityContext); + UserGroupInformation user = HttpUserGroupInformation.get(); KMSClientProvider.checkNotEmpty(name, "name"); KMSWebApp.getKeyCallsMeter().mark(); assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSIONS, name); - Object json = KMSServerJSONUtils.toJSON(provider.getKeyVersions(name)); + + List ret = user.doAs( + new PrivilegedExceptionAction>() { + @Override + public List run() throws Exception { + return provider.getKeyVersions(name); + } + } + ); + + Object json = KMSServerJSONUtils.toJSON(ret); kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, ""); return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java index 58e91475f73..a6c5bf4c2a5 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java @@ -113,8 +113,7 @@ public class KMSACLs implements Runnable { return conf; } - public boolean hasAccess(Type type, String user) { - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + public boolean hasAccess(Type type, UserGroupInformation ugi) { return acls.get(type).isUserAllowed(ugi); } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java index 30d340d785a..dc55a8459cf 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.crypto.key.kms.server; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +30,6 @@ import com.google.common.cache.RemovalNotification; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.security.Principal; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -186,22 +186,22 @@ public class KMSAudit { } } - public void ok(Principal user, KMS.KMSOp op, String key, + public void ok(UserGroupInformation user, KMS.KMSOp op, String key, String extraMsg) { - op(OpStatus.OK, op, user.getName(), key, extraMsg); + op(OpStatus.OK, op, user.getShortUserName(), key, extraMsg); } - public void ok(Principal user, KMS.KMSOp op, String extraMsg) { - op(OpStatus.OK, op, user.getName(), null, extraMsg); + public void ok(UserGroupInformation user, KMS.KMSOp op, String extraMsg) { + op(OpStatus.OK, op, user.getShortUserName(), null, extraMsg); } - public void unauthorized(Principal user, KMS.KMSOp op, String key) { - op(OpStatus.UNAUTHORIZED, op, user.getName(), key, ""); + public void unauthorized(UserGroupInformation user, KMS.KMSOp op, String key) { + op(OpStatus.UNAUTHORIZED, op, user.getShortUserName(), key, ""); } - public void error(Principal user, String method, String url, + public void error(UserGroupInformation user, String method, String url, String extraMsg) { - op(OpStatus.ERROR, null, user.getName(), null, "Method:'" + method + op(OpStatus.ERROR, null, user.getShortUserName(), null, "Method:'" + method + "' Exception:'" + extraMsg + "'"); } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index db60b097ee7..76c22c4646d 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -19,7 +19,13 @@ package org.apache.hadoop.crypto.key.kms.server; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -38,7 +44,8 @@ import java.util.Properties; * file. */ @InterfaceAudience.Private -public class KMSAuthenticationFilter extends AuthenticationFilter { +public class KMSAuthenticationFilter + extends DelegationTokenAuthenticationFilter { private static final String CONF_PREFIX = KMSConfiguration.CONFIG_PREFIX + "authentication."; @@ -55,6 +62,16 @@ public class KMSAuthenticationFilter extends AuthenticationFilter { props.setProperty(name, value); } } + String authType = props.getProperty(AUTH_TYPE); + if (authType.equals(PseudoAuthenticationHandler.TYPE)) { + props.setProperty(AUTH_TYPE, + PseudoDelegationTokenAuthenticationHandler.class.getName()); + } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) { + props.setProperty(AUTH_TYPE, + KerberosDelegationTokenAuthenticationHandler.class.getName()); + } + props.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND, + KMSClientProvider.TOKEN_KIND); return props; } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java index bf24ed8a108..059d7f063c1 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java @@ -23,6 +23,7 @@ import com.sun.jersey.api.container.ContainerException; import org.apache.hadoop.crypto.key.kms.KMSRESTConstants; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authorize.AuthorizationException; import org.slf4j.Logger; @@ -34,7 +35,6 @@ import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; import java.io.IOException; -import java.security.Principal; import java.util.LinkedHashMap; import java.util.Map; @@ -102,7 +102,7 @@ public class KMSExceptionsProvider implements ExceptionMapper { status = Response.Status.INTERNAL_SERVER_ERROR; } if (doAudit) { - KMSWebApp.getKMSAudit().error(KMSMDCFilter.getPrincipal(), + KMSWebApp.getKMSAudit().error(KMSMDCFilter.getUgi(), KMSMDCFilter.getMethod(), KMSMDCFilter.getURL(), getOneLineMessage(exception)); } @@ -110,11 +110,11 @@ public class KMSExceptionsProvider implements ExceptionMapper { } protected void log(Response.Status status, Throwable ex) { - Principal principal = KMSMDCFilter.getPrincipal(); + UserGroupInformation ugi = KMSMDCFilter.getUgi(); String method = KMSMDCFilter.getMethod(); String url = KMSMDCFilter.getURL(); String msg = getOneLineMessage(ex); - LOG.warn("User:{} Method:{} URL:{} Response:{}-{}", principal, method, url, + LOG.warn("User:'{}' Method:{} URL:{} Response:{}-{}", ugi, method, url, status, msg, ex); } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java index 8d24c7ce476..2a3c14971b2 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java @@ -18,6 +18,8 @@ package org.apache.hadoop.crypto.key.kms.server; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -27,7 +29,6 @@ import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.security.Principal; /** * Servlet filter that captures context of the HTTP request to be use in the @@ -37,12 +38,12 @@ import java.security.Principal; public class KMSMDCFilter implements Filter { private static class Data { - private Principal principal; + private UserGroupInformation ugi; private String method; private StringBuffer url; - private Data(Principal principal, String method, StringBuffer url) { - this.principal = principal; + private Data(UserGroupInformation ugi, String method, StringBuffer url) { + this.ugi = ugi; this.method = method; this.url = url; } @@ -50,8 +51,8 @@ public class KMSMDCFilter implements Filter { private static ThreadLocal DATA_TL = new ThreadLocal(); - public static Principal getPrincipal() { - return DATA_TL.get().principal; + public static UserGroupInformation getUgi() { + return DATA_TL.get().ugi; } public static String getMethod() { @@ -72,14 +73,14 @@ public class KMSMDCFilter implements Filter { throws IOException, ServletException { try { DATA_TL.remove(); - Principal principal = ((HttpServletRequest) request).getUserPrincipal(); + UserGroupInformation ugi = HttpUserGroupInformation.get(); String method = ((HttpServletRequest) request).getMethod(); StringBuffer requestURL = ((HttpServletRequest) request).getRequestURL(); String queryString = ((HttpServletRequest) request).getQueryString(); if (queryString != null) { requestURL.append("?").append(queryString); } - DATA_TL.set(new Data(principal, method, requestURL)); + DATA_TL.set(new Data(ugi, method, requestURL)); chain.doFilter(request, response); } finally { DATA_TL.remove(); diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm index ebfe8e2c170..0a1cba52e13 100644 --- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm +++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm @@ -319,6 +319,46 @@ $ keytool -genkey -alias tomcat -keyalg RSA +---+ +** KMS Delegation Token Configuration + + KMS delegation token secret manager can be configured with the following + properties: + + +---+ + + hadoop.kms.authentication.delegation-token.update-interval.sec + 86400 + + How often the master key is rotated, in seconds. Default value 1 day. + + + + + hadoop.kms.authentication.delegation-token.max-lifetime.sec + 604800 + + Maximum lifetime of a delagation token, in seconds. Default value 7 days. + + + + + hadoop.kms.authentication.delegation-token.renew-interval.sec + 86400 + + Renewal interval of a delagation token, in seconds. Default value 1 day. + + + + + hadoop.kms.authentication.delegation-token.removal-scan-interval.sec + 3600 + + Scan interval to remove expired delegation tokens. + + + +---+ + + ** KMS HTTP REST API *** Create a Key diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 021f3cb0533..971c6179b2c 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -22,8 +22,13 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; +import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.io.Text; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.junit.AfterClass; @@ -45,6 +50,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.Writer; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.ServerSocket; import java.net.SocketTimeoutException; @@ -565,6 +571,17 @@ public class TestKMS { Assert.assertEquals("d", meta.getDescription()); Assert.assertEquals(attributes, meta.getAttributes()); + KeyProviderDelegationTokenExtension kpdte = + KeyProviderDelegationTokenExtension. + createKeyProviderDelegationTokenExtension(kp); + Credentials credentials = new Credentials(); + kpdte.addDelegationTokens("foo", credentials); + Assert.assertEquals(1, credentials.getAllTokens().size()); + InetSocketAddress kmsAddr = new InetSocketAddress(getKMSUrl().getHost(), + getKMSUrl().getPort()); + + Assert.assertEquals(new Text("kms-dt"), credentials.getToken( + SecurityUtil.buildTokenService(kmsAddr)).getKind()); return null; } }); @@ -596,13 +613,13 @@ public class TestKMS { public Void call() throws Exception { final Configuration conf = new Configuration(); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128); - URI uri = createKMSUri(getKMSUrl()); - final KeyProvider kp = new KMSClientProvider(uri, conf); + final URI uri = createKMSUri(getKMSUrl()); //nothing allowed doAs("client", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { kp.createKey("k", new KeyProvider.Options(conf)); Assert.fail(); @@ -693,6 +710,7 @@ public class TestKMS { doAs("CREATE", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { KeyProvider.KeyVersion kv = kp.createKey("k0", new KeyProvider.Options(conf)); @@ -707,6 +725,7 @@ public class TestKMS { doAs("DELETE", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { kp.deleteKey("k0"); } catch (Exception ex) { @@ -719,6 +738,7 @@ public class TestKMS { doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { KeyProvider.KeyVersion kv = kp.createKey("k1", new byte[16], new KeyProvider.Options(conf)); @@ -733,6 +753,7 @@ public class TestKMS { doAs("ROLLOVER", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { KeyProvider.KeyVersion kv = kp.rollNewVersion("k1"); Assert.assertNull(kv.getMaterial()); @@ -746,6 +767,7 @@ public class TestKMS { doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { KeyProvider.KeyVersion kv = kp.rollNewVersion("k1", new byte[16]); @@ -761,6 +783,7 @@ public class TestKMS { doAs("GET", new PrivilegedExceptionAction() { @Override public KeyVersion run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { kp.getKeyVersion("k1@0"); KeyVersion kv = kp.getCurrentKey("k1"); @@ -777,6 +800,7 @@ public class TestKMS { new PrivilegedExceptionAction() { @Override public EncryptedKeyVersion run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension. createKeyProviderCryptoExtension(kp); @@ -793,6 +817,7 @@ public class TestKMS { doAs("DECRYPT_EEK", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension. createKeyProviderCryptoExtension(kp); @@ -807,6 +832,7 @@ public class TestKMS { doAs("GET_KEYS", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { kp.getKeys(); } catch (Exception ex) { @@ -819,6 +845,7 @@ public class TestKMS { doAs("GET_METADATA", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); try { kp.getMetadata("k1"); kp.getKeysMetadata("k1"); @@ -836,6 +863,7 @@ public class TestKMS { Thread.sleep(10); // to ensure the ACLs file modifiedTime is newer conf.set(KMSACLs.Type.CREATE.getConfigKey(), "foo"); writeConf(testDir, conf); + Thread.sleep(1000); KMSWebApp.getACLs().run(); // forcing a reload by hand. @@ -844,6 +872,7 @@ public class TestKMS { @Override public Void run() throws Exception { try { + KeyProvider kp = new KMSClientProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("k2", new KeyProvider.Options(conf)); Assert.fail(); @@ -982,4 +1011,69 @@ public class TestKMS { sock.close(); } + + @Test + public void testDelegationTokenAccess() throws Exception { + final File testDir = getTestDir(); + Configuration conf = createBaseKMSConf(testDir); + conf.set("hadoop.kms.authentication.type", "kerberos"); + conf.set("hadoop.kms.authentication.kerberos.keytab", + keytab.getAbsolutePath()); + conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost"); + conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT"); + + writeConf(testDir, conf); + + runServer(null, null, testDir, new KMSCallable() { + @Override + public Void call() throws Exception { + final Configuration conf = new Configuration(); + conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 64); + final URI uri = createKMSUri(getKMSUrl()); + final Credentials credentials = new Credentials(); + final UserGroupInformation nonKerberosUgi = + UserGroupInformation.getCurrentUser(); + + try { + KeyProvider kp = new KMSClientProvider(uri, conf); + kp.createKey("kA", new KeyProvider.Options(conf)); + } catch (IOException ex) { + System.out.println(ex.getMessage()); + } + + doAs("client", new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); + KeyProviderDelegationTokenExtension kpdte = + KeyProviderDelegationTokenExtension. + createKeyProviderDelegationTokenExtension(kp); + kpdte.addDelegationTokens("foo", credentials); + return null; + } + }); + + nonKerberosUgi.addCredentials(credentials); + + try { + KeyProvider kp = new KMSClientProvider(uri, conf); + kp.createKey("kA", new KeyProvider.Options(conf)); + } catch (IOException ex) { + System.out.println(ex.getMessage()); + } + + nonKerberosUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + KeyProvider kp = new KMSClientProvider(uri, conf); + kp.createKey("kD", new KeyProvider.Options(conf)); + return null; + } + }); + + return null; + } + }); + } + } diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java index e65a1027ea7..7c0ad3bc9d1 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java @@ -18,6 +18,7 @@ package org.apache.hadoop.crypto.key.kms.server; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.Assert; import org.junit.Test; @@ -27,7 +28,8 @@ public class TestKMSACLs { public void testDefaults() { KMSACLs acls = new KMSACLs(new Configuration(false)); for (KMSACLs.Type type : KMSACLs.Type.values()) { - Assert.assertTrue(acls.hasAccess(type, "foo")); + Assert.assertTrue(acls.hasAccess(type, + UserGroupInformation.createRemoteUser("foo"))); } } @@ -39,8 +41,10 @@ public class TestKMSACLs { } KMSACLs acls = new KMSACLs(conf); for (KMSACLs.Type type : KMSACLs.Type.values()) { - Assert.assertTrue(acls.hasAccess(type, type.toString())); - Assert.assertFalse(acls.hasAccess(type, "foo")); + Assert.assertTrue(acls.hasAccess(type, + UserGroupInformation.createRemoteUser(type.toString()))); + Assert.assertFalse(acls.hasAccess(type, + UserGroupInformation.createRemoteUser("foo"))); } } diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java index 857884d4e14..f4962693c16 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java @@ -21,9 +21,9 @@ import java.io.ByteArrayOutputStream; import java.io.FilterOutputStream; import java.io.OutputStream; import java.io.PrintStream; -import java.security.Principal; import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.LogManager; import org.apache.log4j.PropertyConfigurator; import org.junit.After; @@ -81,8 +81,8 @@ public class TestKMSAudit { @Test public void testAggregation() throws Exception { - Principal luser = Mockito.mock(Principal.class); - Mockito.when(luser.getName()).thenReturn("luser"); + UserGroupInformation luser = Mockito.mock(UserGroupInformation.class); + Mockito.when(luser.getShortUserName()).thenReturn("luser"); kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg"); kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg"); kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg"); @@ -109,8 +109,8 @@ public class TestKMSAudit { @Test public void testAggregationUnauth() throws Exception { - Principal luser = Mockito.mock(Principal.class); - Mockito.when(luser.getName()).thenReturn("luser"); + UserGroupInformation luser = Mockito.mock(UserGroupInformation.class); + Mockito.when(luser.getShortUserName()).thenReturn("luser"); kmsAudit.unauthorized(luser, KMSOp.GENERATE_EEK, "k2"); Thread.sleep(1000); kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg"); From be117cbcdf9e7c72bb24aaf3556cf1212963ed4b Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 15 Aug 2014 05:45:14 +0000 Subject: [PATCH 45/67] HADOOP-10967. Improve DefaultCryptoExtension#generateEncryptedKey performance. (hitliuyi via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618101 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../crypto/key/KeyProviderCryptoExtension.java | 13 ++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 0a17758483c..46784c17128 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -438,6 +438,9 @@ Trunk (Unreleased) HADOOP-10121. Fix javadoc spelling for HadoopArchives#writeTopLevelDirs (Akira AJISAKA via aw) + HADOOP-10967. Improve DefaultCryptoExtension#generateEncryptedKey + performance. (hitliuyi via tucu) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java index e2b25ae8ed6..026f285f4c4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java @@ -219,6 +219,13 @@ public class KeyProviderCryptoExtension extends private static class DefaultCryptoExtension implements CryptoExtension { private final KeyProvider keyProvider; + private static final ThreadLocal RANDOM = + new ThreadLocal() { + @Override + protected SecureRandom initialValue() { + return new SecureRandom(); + } + }; private DefaultCryptoExtension(KeyProvider keyProvider) { this.keyProvider = keyProvider; @@ -233,10 +240,10 @@ public class KeyProviderCryptoExtension extends "No KeyVersion exists for key '%s' ", encryptionKeyName); // Generate random bytes for new key and IV Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); - SecureRandom random = SecureRandom.getInstance("SHA1PRNG"); final byte[] newKey = new byte[encryptionKey.getMaterial().length]; - random.nextBytes(newKey); - final byte[] iv = random.generateSeed(cipher.getBlockSize()); + RANDOM.get().nextBytes(newKey); + final byte[] iv = new byte[cipher.getBlockSize()]; + RANDOM.get().nextBytes(iv); // Encryption key IV is derived from new key's IV final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv); // Encrypt the new key From cfeaf4cd093a83db5c84ce04a4438a2a60663df9 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 15 Aug 2014 05:50:00 +0000 Subject: [PATCH 46/67] HADOOP-10964. Small fix for NetworkTopologyWithNodeGroup#sortByDistance. Contributed by Yi Liu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618103 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 46784c17128..8cd6a359f2f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -594,6 +594,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10843. TestGridmixRecord unit tests failure on PowerPC (Jinghui Wang via Colin Patrick McCabe) + HADOOP-10964. Small fix for NetworkTopologyWithNodeGroup#sortByDistance. + (Yi Liu via wang) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java index 86d290abd60..cc598c0986f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java @@ -293,7 +293,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology { return; } } - super.sortByDistance(reader, nodes, nodes.length, seed, + super.sortByDistance(reader, nodes, activeLen, seed, randomizeBlockLocationsPerBlock); } From 16b7bd4e6cb988762e5afe3b23fc98a8ddd807e1 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 15 Aug 2014 05:53:03 +0000 Subject: [PATCH 47/67] Move HADOOP-10121 to correct section of CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618105 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 8cd6a359f2f..aec02b4f61d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -435,9 +435,6 @@ Trunk (Unreleased) HADOOP-10862. Miscellaneous trivial corrections to KMS classes. (asuresh via tucu) - HADOOP-10121. Fix javadoc spelling for HadoopArchives#writeTopLevelDirs - (Akira AJISAKA via aw) - HADOOP-10967. Improve DefaultCryptoExtension#generateEncryptedKey performance. (hitliuyi via tucu) @@ -594,6 +591,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10843. TestGridmixRecord unit tests failure on PowerPC (Jinghui Wang via Colin Patrick McCabe) + HADOOP-10121. Fix javadoc spelling for HadoopArchives#writeTopLevelDirs + (Akira AJISAKA via aw) + HADOOP-10964. Small fix for NetworkTopologyWithNodeGroup#sortByDistance. (Yi Liu via wang) From 7360cec692be5dcc3377ae5082fe22870caac96b Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 15 Aug 2014 06:00:31 +0000 Subject: [PATCH 48/67] YARN-2378. Added support for moving applications across queues in CapacityScheduler. Contributed by Subramaniam Venkatraman Krishnan git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618106 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/rmapp/RMAppImpl.java | 19 +- .../scheduler/AbstractYarnScheduler.java | 31 + .../scheduler/AppSchedulingInfo.java | 3 +- .../scheduler/YarnScheduler.java | 10 + .../scheduler/capacity/CSQueue.java | 18 + .../scheduler/capacity/CapacityScheduler.java | 58 ++ .../scheduler/capacity/LeafQueue.java | 41 +- .../scheduler/capacity/ParentQueue.java | 33 + .../capacity/TestCapacityScheduler.java | 784 ++++++++++++++++++ 10 files changed, 989 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b879600537b..d0d32958f1e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -50,6 +50,9 @@ Release 2.6.0 - UNRELEASED YARN-2277. Added cross-origin support for the timeline server web services. (Jonathan Eagles via zjshen) + YARN-2378. Added support for moving applications across queues in + CapacityScheduler. (Subramaniam Venkatraman Krishnan via jianhe) + IMPROVEMENTS YARN-2242. Improve exception information on AM launch crashes. (Li Lu diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 45c89592af8..48cf4602991 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -166,6 +166,8 @@ public class RMAppImpl implements RMApp, Recoverable { RMAppEventType.APP_REJECTED, new FinalSavingTransition(new AppRejectedTransition(), RMAppState.FAILED)) + .addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING, + RMAppEventType.MOVE, new RMAppMoveTransition()) // Transitions from SUBMITTED state .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, @@ -243,7 +245,7 @@ public class RMAppImpl implements RMApp, Recoverable { // ignorable transitions .addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING, EnumSet.of(RMAppEventType.NODE_UPDATE, RMAppEventType.KILL, - RMAppEventType.APP_NEW_SAVED)) + RMAppEventType.APP_NEW_SAVED, RMAppEventType.MOVE)) // Transitions from FINISHING state .addTransition(RMAppState.FINISHING, RMAppState.FINISHED, @@ -254,9 +256,9 @@ public class RMAppImpl implements RMApp, Recoverable { // ignorable transitions .addTransition(RMAppState.FINISHING, RMAppState.FINISHING, EnumSet.of(RMAppEventType.NODE_UPDATE, - // ignore Kill as we have already saved the final Finished state in - // state store. - RMAppEventType.KILL)) + // ignore Kill/Move as we have already saved the final Finished state + // in state store. + RMAppEventType.KILL, RMAppEventType.MOVE)) // Transitions from KILLING state .addTransition(RMAppState.KILLING, RMAppState.KILLING, @@ -274,7 +276,7 @@ public class RMAppImpl implements RMApp, Recoverable { RMAppEventType.ATTEMPT_FINISHED, RMAppEventType.ATTEMPT_FAILED, RMAppEventType.APP_UPDATE_SAVED, - RMAppEventType.KILL)) + RMAppEventType.KILL, RMAppEventType.MOVE)) // Transitions from FINISHED state // ignorable transitions @@ -286,7 +288,7 @@ public class RMAppImpl implements RMApp, Recoverable { RMAppEventType.NODE_UPDATE, RMAppEventType.ATTEMPT_UNREGISTERED, RMAppEventType.ATTEMPT_FINISHED, - RMAppEventType.KILL)) + RMAppEventType.KILL, RMAppEventType.MOVE)) // Transitions from FAILED state // ignorable transitions @@ -294,7 +296,8 @@ public class RMAppImpl implements RMApp, Recoverable { RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) .addTransition(RMAppState.FAILED, RMAppState.FAILED, - EnumSet.of(RMAppEventType.KILL, RMAppEventType.NODE_UPDATE)) + EnumSet.of(RMAppEventType.KILL, RMAppEventType.NODE_UPDATE, + RMAppEventType.MOVE)) // Transitions from KILLED state // ignorable transitions @@ -307,7 +310,7 @@ public class RMAppImpl implements RMApp, Recoverable { EnumSet.of(RMAppEventType.APP_ACCEPTED, RMAppEventType.APP_REJECTED, RMAppEventType.KILL, RMAppEventType.ATTEMPT_FINISHED, RMAppEventType.ATTEMPT_FAILED, - RMAppEventType.NODE_UPDATE)) + RMAppEventType.NODE_UPDATE, RMAppEventType.MOVE)) .installTopology(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 5764c8c7a65..e5f22b8fc41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; @@ -48,6 +50,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.util.resource.Resources; +import com.google.common.util.concurrent.SettableFuture; + @SuppressWarnings("unchecked") public abstract class AbstractYarnScheduler @@ -317,4 +321,31 @@ public abstract class AbstractYarnScheduler public SchedulerNode getSchedulerNode(NodeId nodeId) { return nodes.get(nodeId); } + + @Override + public synchronized void moveAllApps(String sourceQueue, String destQueue) + throws YarnException { + // check if destination queue is a valid leaf queue + try { + getQueueInfo(destQueue, false, false); + } catch (IOException e) { + LOG.warn(e); + throw new YarnException(e); + } + // check if source queue is a valid + List apps = getAppsInQueue(sourceQueue); + if (apps == null) { + String errMsg = "The specified Queue: " + sourceQueue + " doesn't exist"; + LOG.warn(errMsg); + throw new YarnException(errMsg); + } + // generate move events for each pending/running app + for (ApplicationAttemptId app : apps) { + SettableFuture future = SettableFuture.create(); + this.rmContext + .getDispatcher() + .getEventHandler() + .handle(new RMAppMoveEvent(app.getApplicationId(), destQueue, future)); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 3c6f210b4b6..e871f429e16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -54,7 +54,7 @@ public class AppSchedulingInfo { private static final Log LOG = LogFactory.getLog(AppSchedulingInfo.class); private final ApplicationAttemptId applicationAttemptId; final ApplicationId applicationId; - private final String queueName; + private String queueName; Queue queue; final String user; // TODO making containerIdCounter long @@ -410,6 +410,7 @@ public class AppSchedulingInfo { activeUsersManager = newQueue.getActiveUsersManager(); activeUsersManager.activateApplication(user, applicationId); this.queue = newQueue; + this.queueName = newQueue.getQueueName(); } synchronized public void stop(RMAppAttemptState rmAppAttemptFinalState) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 21eba396985..79dc773b410 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -202,4 +202,14 @@ public interface YarnScheduler extends EventHandler { @Evolving public String moveApplication(ApplicationId appId, String newQueue) throws YarnException; + + /** + * Completely drain sourceQueue of applications, by moving all of them to + * destQueue. + * + * @param sourceQueue + * @param destQueue + * @throws YarnException + */ + void moveAllApps(String sourceQueue, String destQueue) throws YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index ccb71e21236..04c2fd51bf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -238,4 +238,22 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue { * @param apps the collection to add the applications to */ public void collectSchedulerApplications(Collection apps); + + /** + * Detach a container from this queue + * @param clusterResource the current cluster resource + * @param application application to which the container was assigned + * @param container the container to detach + */ + public void detachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer container); + + /** + * Attach a container to this queue + * @param clusterResource the current cluster resource + * @param application application to which the container was assigned + * @param container the container to attach + */ + public void attachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer container); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 65ff81c0216..1b31a1a6bbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; @@ -547,6 +548,8 @@ public class CapacityScheduler extends .handle(new RMAppRejectedEvent(applicationId, ace.toString())); return; } + // update the metrics + queue.getMetrics().submitApp(user); SchedulerApplication application = new SchedulerApplication(queue, user); applications.put(applicationId, application); @@ -1131,4 +1134,59 @@ public class CapacityScheduler extends throw new IOException(e); } } + + @Override + public synchronized String moveApplication(ApplicationId appId, + String targetQueueName) throws YarnException { + FiCaSchedulerApp app = + getApplicationAttempt(ApplicationAttemptId.newInstance(appId, 0)); + String sourceQueueName = app.getQueue().getQueueName(); + LeafQueue source = getAndCheckLeafQueue(sourceQueueName); + LeafQueue dest = getAndCheckLeafQueue(targetQueueName); + // Validation check - ACLs, submission limits for user & queue + String user = app.getUser(); + try { + dest.submitApplication(appId, user, targetQueueName); + } catch (AccessControlException e) { + throw new YarnException(e); + } + // Move all live containers + for (RMContainer rmContainer : app.getLiveContainers()) { + source.detachContainer(clusterResource, app, rmContainer); + // attach the Container to another queue + dest.attachContainer(clusterResource, app, rmContainer); + } + // Detach the application.. + source.finishApplicationAttempt(app, sourceQueueName); + source.getParent().finishApplication(appId, app.getUser()); + // Finish app & update metrics + app.move(dest); + // Submit to a new queue + dest.submitApplicationAttempt(app, user); + applications.get(appId).setQueue(dest); + LOG.info("App: " + app.getApplicationId() + " successfully moved from " + + sourceQueueName + " to: " + targetQueueName); + return targetQueueName; + } + + /** + * Check that the String provided in input is the name of an existing, + * LeafQueue, if successful returns the queue. + * + * @param queue + * @return the LeafQueue + * @throws YarnException + */ + private LeafQueue getAndCheckLeafQueue(String queue) throws YarnException { + CSQueue ret = this.getQueue(queue); + if (ret == null) { + throw new YarnException("The specified Queue: " + queue + + " doesn't exist"); + } + if (!(ret instanceof LeafQueue)) { + throw new YarnException("The specified Queue: " + queue + + " is not a Leaf Queue. Move is supported only for Leaf Queues."); + } + return (LeafQueue) ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 4fd8b490577..5c93c5fc026 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -643,7 +643,10 @@ public class LeafQueue implements CSQueue { addApplicationAttempt(application, user); } - metrics.submitAppAttempt(userName); + // We don't want to update metrics for move app + if (application.isPending()) { + metrics.submitAppAttempt(userName); + } getParent().submitApplicationAttempt(application, userName); } @@ -701,7 +704,6 @@ public class LeafQueue implements CSQueue { throw ace; } - metrics.submitApp(userName); } private synchronized void activateApplications() { @@ -1620,8 +1622,43 @@ public class LeafQueue implements CSQueue { @Override public void collectSchedulerApplications( Collection apps) { + for (FiCaSchedulerApp pendingApp : pendingApplications) { + apps.add(pendingApp.getApplicationAttemptId()); + } for (FiCaSchedulerApp app : activeApplications) { apps.add(app.getApplicationAttemptId()); } } + + @Override + public void attachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer rmContainer) { + if (application != null) { + allocateResource(clusterResource, application, rmContainer.getContainer() + .getResource()); + LOG.info("movedContainer" + " container=" + rmContainer.getContainer() + + " resource=" + rmContainer.getContainer().getResource() + + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + + usedResources + " cluster=" + clusterResource); + // Inform the parent queue + getParent().attachContainer(clusterResource, application, rmContainer); + } + } + + @Override + public void detachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer rmContainer) { + if (application != null) { + releaseResource(clusterResource, application, rmContainer.getContainer() + .getResource()); + LOG.info("movedContainer" + " container=" + rmContainer.getContainer() + + " resource=" + rmContainer.getContainer().getResource() + + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + + usedResources + " cluster=" + clusterResource); + // Inform the parent queue + getParent().detachContainer(clusterResource, application, rmContainer); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index d83eed34cb3..8c654b7ded1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -791,4 +791,37 @@ public class ParentQueue implements CSQueue { queue.collectSchedulerApplications(apps); } } + + @Override + public void attachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer rmContainer) { + if (application != null) { + allocateResource(clusterResource, rmContainer.getContainer() + .getResource()); + LOG.info("movedContainer" + " queueMoveIn=" + getQueueName() + + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster=" + + clusterResource); + // Inform the parent + if (parent != null) { + parent.attachContainer(clusterResource, application, rmContainer); + } + } + } + + @Override + public void detachContainer(Resource clusterResource, + FiCaSchedulerApp application, RMContainer rmContainer) { + if (application != null) { + releaseResource(clusterResource, rmContainer.getContainer().getResource()); + LOG.info("movedContainer" + " queueMoveOut=" + getQueueName() + + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster=" + + clusterResource); + // Inform the parent + if (parent != null) { + parent.detachContainer(clusterResource, application, rmContainer); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 0efd48fa28d..45c4950202d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -68,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -100,6 +102,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -1014,4 +1020,782 @@ public class TestCapacityScheduler { // Now with updated ResourceRequest, a container is allocated for AM. Assert.assertTrue(containers.size() == 1); } + + private MockRM setUpMove() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(conf); + rm.start(); + return rm; + } + + @Test + public void testMoveAppBasic() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + String queue = + scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() + .getQueueName(); + Assert.assertTrue(queue.equals("a1")); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + List appsInB1 = scheduler.getAppsInQueue("b1"); + assertTrue(appsInB1.isEmpty()); + + List appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.isEmpty()); + + // now move the app + scheduler.moveApplication(app.getApplicationId(), "b1"); + + // check postconditions + appsInB1 = scheduler.getAppsInQueue("b1"); + assertEquals(1, appsInB1.size()); + queue = + scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() + .getQueueName(); + System.out.println(queue); + Assert.assertTrue(queue.equals("b1")); + + appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInB.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + appsInA1 = scheduler.getAppsInQueue("a1"); + assertTrue(appsInA1.isEmpty()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.isEmpty()); + + rm.stop(); + } + + @Test + public void testMoveAppSameParent() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + String queue = + scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() + .getQueueName(); + Assert.assertTrue(queue.equals("a1")); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + List appsInA2 = scheduler.getAppsInQueue("a2"); + assertTrue(appsInA2.isEmpty()); + + // now move the app + scheduler.moveApplication(app.getApplicationId(), "a2"); + + // check postconditions + appsInA2 = scheduler.getAppsInQueue("a2"); + assertEquals(1, appsInA2.size()); + queue = + scheduler.getApplicationAttempt(appsInA2.get(0)).getQueue() + .getQueueName(); + Assert.assertTrue(queue.equals("a2")); + + appsInA1 = scheduler.getAppsInQueue("a1"); + assertTrue(appsInA1.isEmpty()); + + appsInA = scheduler.getAppsInQueue("a"); + assertEquals(1, appsInA.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + rm.stop(); + } + + @Test + public void testMoveAppForMoveToQueueWithFreeCap() throws Exception { + + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + // Register node1 + String host_0 = "host_0"; + NodeManager nm_0 = + registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(4 * GB, 1)); + + // Register node2 + String host_1 = "host_1"; + NodeManager nm_1 = + registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(2 * GB, 1)); + + // ResourceRequest priorities + Priority priority_0 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(0); + Priority priority_1 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(1); + + // Submit application_0 + Application application_0 = + new Application("user_0", "a1", resourceManager); + application_0.submit(); // app + app attempt event sent to scheduler + + application_0.addNodeManager(host_0, 1234, nm_0); + application_0.addNodeManager(host_1, 1234, nm_1); + + Resource capability_0_0 = Resources.createResource(1 * GB, 1); + application_0.addResourceRequestSpec(priority_1, capability_0_0); + + Resource capability_0_1 = Resources.createResource(2 * GB, 1); + application_0.addResourceRequestSpec(priority_0, capability_0_1); + + Task task_0_0 = + new Task(application_0, priority_1, new String[] { host_0, host_1 }); + application_0.addTask(task_0_0); + + // Submit application_1 + Application application_1 = + new Application("user_1", "b2", resourceManager); + application_1.submit(); // app + app attempt event sent to scheduler + + application_1.addNodeManager(host_0, 1234, nm_0); + application_1.addNodeManager(host_1, 1234, nm_1); + + Resource capability_1_0 = Resources.createResource(1 * GB, 1); + application_1.addResourceRequestSpec(priority_1, capability_1_0); + + Resource capability_1_1 = Resources.createResource(2 * GB, 1); + application_1.addResourceRequestSpec(priority_0, capability_1_1); + + Task task_1_0 = + new Task(application_1, priority_1, new String[] { host_0, host_1 }); + application_1.addTask(task_1_0); + + // Send resource requests to the scheduler + application_0.schedule(); // allocate + application_1.schedule(); // allocate + + // task_0_0 task_1_0 allocated, used=2G + nodeUpdate(nm_0); + + // nothing allocated + nodeUpdate(nm_1); + + // Get allocations from the scheduler + application_0.schedule(); // task_0_0 + checkApplicationResourceUsage(1 * GB, application_0); + + application_1.schedule(); // task_1_0 + checkApplicationResourceUsage(1 * GB, application_1); + + checkNodeResourceUsage(2 * GB, nm_0); // task_0_0 (1G) and task_1_0 (1G) 2G + // available + checkNodeResourceUsage(0 * GB, nm_1); // no tasks, 2G available + + // move app from a1(30% cap of total 10.5% cap) to b1(79,2% cap of 89,5% + // total cap) + scheduler.moveApplication(application_0.getApplicationId(), "b1"); + + // 2GB 1C + Task task_1_1 = + new Task(application_1, priority_0, + new String[] { ResourceRequest.ANY }); + application_1.addTask(task_1_1); + + application_1.schedule(); + + // 2GB 1C + Task task_0_1 = + new Task(application_0, priority_0, new String[] { host_0, host_1 }); + application_0.addTask(task_0_1); + + application_0.schedule(); + + // prev 2G used free 2G + nodeUpdate(nm_0); + + // prev 0G used free 2G + nodeUpdate(nm_1); + + // Get allocations from the scheduler + application_1.schedule(); + checkApplicationResourceUsage(3 * GB, application_1); + + // Get allocations from the scheduler + application_0.schedule(); + checkApplicationResourceUsage(3 * GB, application_0); + + checkNodeResourceUsage(4 * GB, nm_0); + checkNodeResourceUsage(2 * GB, nm_1); + + } + + @Test + public void testMoveAppSuccess() throws Exception { + + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + // Register node1 + String host_0 = "host_0"; + NodeManager nm_0 = + registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1)); + + // Register node2 + String host_1 = "host_1"; + NodeManager nm_1 = + registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1)); + + // ResourceRequest priorities + Priority priority_0 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(0); + Priority priority_1 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(1); + + // Submit application_0 + Application application_0 = + new Application("user_0", "a1", resourceManager); + application_0.submit(); // app + app attempt event sent to scheduler + + application_0.addNodeManager(host_0, 1234, nm_0); + application_0.addNodeManager(host_1, 1234, nm_1); + + Resource capability_0_0 = Resources.createResource(3 * GB, 1); + application_0.addResourceRequestSpec(priority_1, capability_0_0); + + Resource capability_0_1 = Resources.createResource(2 * GB, 1); + application_0.addResourceRequestSpec(priority_0, capability_0_1); + + Task task_0_0 = + new Task(application_0, priority_1, new String[] { host_0, host_1 }); + application_0.addTask(task_0_0); + + // Submit application_1 + Application application_1 = + new Application("user_1", "b2", resourceManager); + application_1.submit(); // app + app attempt event sent to scheduler + + application_1.addNodeManager(host_0, 1234, nm_0); + application_1.addNodeManager(host_1, 1234, nm_1); + + Resource capability_1_0 = Resources.createResource(1 * GB, 1); + application_1.addResourceRequestSpec(priority_1, capability_1_0); + + Resource capability_1_1 = Resources.createResource(2 * GB, 1); + application_1.addResourceRequestSpec(priority_0, capability_1_1); + + Task task_1_0 = + new Task(application_1, priority_1, new String[] { host_0, host_1 }); + application_1.addTask(task_1_0); + + // Send resource requests to the scheduler + application_0.schedule(); // allocate + application_1.schedule(); // allocate + + // b2 can only run 1 app at a time + scheduler.moveApplication(application_0.getApplicationId(), "b2"); + + nodeUpdate(nm_0); + + nodeUpdate(nm_1); + + // Get allocations from the scheduler + application_0.schedule(); // task_0_0 + checkApplicationResourceUsage(0 * GB, application_0); + + application_1.schedule(); // task_1_0 + checkApplicationResourceUsage(1 * GB, application_1); + + // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is + // not scheduled + checkNodeResourceUsage(1 * GB, nm_0); + checkNodeResourceUsage(0 * GB, nm_1); + + // lets move application_0 to a queue where it can run + scheduler.moveApplication(application_0.getApplicationId(), "a2"); + application_0.schedule(); + + nodeUpdate(nm_1); + + // Get allocations from the scheduler + application_0.schedule(); // task_0_0 + checkApplicationResourceUsage(3 * GB, application_0); + + checkNodeResourceUsage(1 * GB, nm_0); + checkNodeResourceUsage(3 * GB, nm_1); + + } + + @Test(expected = YarnException.class) + public void testMoveAppViolateQueueState() throws Exception { + + resourceManager = new ResourceManager(); + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + StringBuilder qState = new StringBuilder(); + qState.append(CapacitySchedulerConfiguration.PREFIX).append(B) + .append(CapacitySchedulerConfiguration.DOT) + .append(CapacitySchedulerConfiguration.STATE); + csConf.set(qState.toString(), QueueState.STOPPED.name()); + YarnConfiguration conf = new YarnConfiguration(csConf); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + resourceManager.init(conf); + resourceManager.getRMContext().getContainerTokenSecretManager() + .rollMasterKey(); + resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); + ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start(); + mockContext = mock(RMContext.class); + when(mockContext.getConfigurationProvider()).thenReturn( + new LocalConfigurationProvider()); + + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + // Register node1 + String host_0 = "host_0"; + NodeManager nm_0 = + registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(6 * GB, 1)); + + // ResourceRequest priorities + Priority priority_0 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(0); + Priority priority_1 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(1); + + // Submit application_0 + Application application_0 = + new Application("user_0", "a1", resourceManager); + application_0.submit(); // app + app attempt event sent to scheduler + + application_0.addNodeManager(host_0, 1234, nm_0); + + Resource capability_0_0 = Resources.createResource(3 * GB, 1); + application_0.addResourceRequestSpec(priority_1, capability_0_0); + + Resource capability_0_1 = Resources.createResource(2 * GB, 1); + application_0.addResourceRequestSpec(priority_0, capability_0_1); + + Task task_0_0 = + new Task(application_0, priority_1, new String[] { host_0 }); + application_0.addTask(task_0_0); + + // Send resource requests to the scheduler + application_0.schedule(); // allocate + + // task_0_0 allocated + nodeUpdate(nm_0); + + // Get allocations from the scheduler + application_0.schedule(); // task_0_0 + checkApplicationResourceUsage(3 * GB, application_0); + + checkNodeResourceUsage(3 * GB, nm_0); + // b2 queue contains 3GB consumption app, + // add another 3GB will hit max capacity limit on queue b + scheduler.moveApplication(application_0.getApplicationId(), "b1"); + + } + + @Test + public void testMoveAppQueueMetricsCheck() throws Exception { + ResourceScheduler scheduler = resourceManager.getResourceScheduler(); + + // Register node1 + String host_0 = "host_0"; + NodeManager nm_0 = + registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1)); + + // Register node2 + String host_1 = "host_1"; + NodeManager nm_1 = + registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, + Resources.createResource(5 * GB, 1)); + + // ResourceRequest priorities + Priority priority_0 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(0); + Priority priority_1 = + org.apache.hadoop.yarn.server.resourcemanager.resource.Priority + .create(1); + + // Submit application_0 + Application application_0 = + new Application("user_0", "a1", resourceManager); + application_0.submit(); // app + app attempt event sent to scheduler + + application_0.addNodeManager(host_0, 1234, nm_0); + application_0.addNodeManager(host_1, 1234, nm_1); + + Resource capability_0_0 = Resources.createResource(3 * GB, 1); + application_0.addResourceRequestSpec(priority_1, capability_0_0); + + Resource capability_0_1 = Resources.createResource(2 * GB, 1); + application_0.addResourceRequestSpec(priority_0, capability_0_1); + + Task task_0_0 = + new Task(application_0, priority_1, new String[] { host_0, host_1 }); + application_0.addTask(task_0_0); + + // Submit application_1 + Application application_1 = + new Application("user_1", "b2", resourceManager); + application_1.submit(); // app + app attempt event sent to scheduler + + application_1.addNodeManager(host_0, 1234, nm_0); + application_1.addNodeManager(host_1, 1234, nm_1); + + Resource capability_1_0 = Resources.createResource(1 * GB, 1); + application_1.addResourceRequestSpec(priority_1, capability_1_0); + + Resource capability_1_1 = Resources.createResource(2 * GB, 1); + application_1.addResourceRequestSpec(priority_0, capability_1_1); + + Task task_1_0 = + new Task(application_1, priority_1, new String[] { host_0, host_1 }); + application_1.addTask(task_1_0); + + // Send resource requests to the scheduler + application_0.schedule(); // allocate + application_1.schedule(); // allocate + + nodeUpdate(nm_0); + + nodeUpdate(nm_1); + + CapacityScheduler cs = + (CapacityScheduler) resourceManager.getResourceScheduler(); + CSQueue origRootQ = cs.getRootQueue(); + CapacitySchedulerInfo oldInfo = new CapacitySchedulerInfo(origRootQ); + int origNumAppsA = getNumAppsInQueue("a", origRootQ.getChildQueues()); + int origNumAppsRoot = origRootQ.getNumApplications(); + + scheduler.moveApplication(application_0.getApplicationId(), "a2"); + + CSQueue newRootQ = cs.getRootQueue(); + int newNumAppsA = getNumAppsInQueue("a", newRootQ.getChildQueues()); + int newNumAppsRoot = newRootQ.getNumApplications(); + CapacitySchedulerInfo newInfo = new CapacitySchedulerInfo(newRootQ); + CapacitySchedulerLeafQueueInfo origOldA1 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", oldInfo.getQueues()); + CapacitySchedulerLeafQueueInfo origNewA1 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", newInfo.getQueues()); + CapacitySchedulerLeafQueueInfo targetOldA2 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", oldInfo.getQueues()); + CapacitySchedulerLeafQueueInfo targetNewA2 = + (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", newInfo.getQueues()); + // originally submitted here + assertEquals(1, origOldA1.getNumApplications()); + assertEquals(1, origNumAppsA); + assertEquals(2, origNumAppsRoot); + // after the move + assertEquals(0, origNewA1.getNumApplications()); + assertEquals(1, newNumAppsA); + assertEquals(2, newNumAppsRoot); + // original consumption on a1 + assertEquals(3 * GB, origOldA1.getResourcesUsed().getMemory()); + assertEquals(1, origOldA1.getResourcesUsed().getvCores()); + assertEquals(0, origNewA1.getResourcesUsed().getMemory()); // after the move + assertEquals(0, origNewA1.getResourcesUsed().getvCores()); // after the move + // app moved here with live containers + assertEquals(3 * GB, targetNewA2.getResourcesUsed().getMemory()); + assertEquals(1, targetNewA2.getResourcesUsed().getvCores()); + // it was empty before the move + assertEquals(0, targetOldA2.getNumApplications()); + assertEquals(0, targetOldA2.getResourcesUsed().getMemory()); + assertEquals(0, targetOldA2.getResourcesUsed().getvCores()); + // after the app moved here + assertEquals(1, targetNewA2.getNumApplications()); + // 1 container on original queue before move + assertEquals(1, origOldA1.getNumContainers()); + // after the move the resource released + assertEquals(0, origNewA1.getNumContainers()); + // and moved to the new queue + assertEquals(1, targetNewA2.getNumContainers()); + // which originally didn't have any + assertEquals(0, targetOldA2.getNumContainers()); + // 1 user with 3GB + assertEquals(3 * GB, origOldA1.getUsers().getUsersList().get(0) + .getResourcesUsed().getMemory()); + // 1 user with 1 core + assertEquals(1, origOldA1.getUsers().getUsersList().get(0) + .getResourcesUsed().getvCores()); + // user ha no more running app in the orig queue + assertEquals(0, origNewA1.getUsers().getUsersList().size()); + // 1 user with 3GB + assertEquals(3 * GB, targetNewA2.getUsers().getUsersList().get(0) + .getResourcesUsed().getMemory()); + // 1 user with 1 core + assertEquals(1, targetNewA2.getUsers().getUsersList().get(0) + .getResourcesUsed().getvCores()); + + // Get allocations from the scheduler + application_0.schedule(); // task_0_0 + checkApplicationResourceUsage(3 * GB, application_0); + + application_1.schedule(); // task_1_0 + checkApplicationResourceUsage(1 * GB, application_1); + + // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is + // not scheduled + checkNodeResourceUsage(4 * GB, nm_0); + checkNodeResourceUsage(0 * GB, nm_1); + + } + + private int getNumAppsInQueue(String name, List queues) { + for (CSQueue queue : queues) { + if (queue.getQueueName().equals(name)) { + return queue.getNumApplications(); + } + } + return -1; + } + + private CapacitySchedulerQueueInfo getQueueInfo(String name, + CapacitySchedulerQueueInfoList info) { + if (info != null) { + for (CapacitySchedulerQueueInfo queueInfo : info.getQueueInfoList()) { + if (queueInfo.getQueueName().equals(name)) { + return queueInfo; + } else { + CapacitySchedulerQueueInfo result = + getQueueInfo(name, queueInfo.getQueues()); + if (result == null) { + continue; + } + return result; + } + } + } + return null; + } + + @Test + public void testMoveAllApps() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + String queue = + scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() + .getQueueName(); + Assert.assertTrue(queue.equals("a1")); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + List appsInB1 = scheduler.getAppsInQueue("b1"); + assertTrue(appsInB1.isEmpty()); + + List appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.isEmpty()); + + // now move the app + scheduler.moveAllApps("a1", "b1"); + + // check postconditions + Thread.sleep(1000); + appsInB1 = scheduler.getAppsInQueue("b1"); + assertEquals(1, appsInB1.size()); + queue = + scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() + .getQueueName(); + Assert.assertTrue(queue.equals("b1")); + + appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInB.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + appsInA1 = scheduler.getAppsInQueue("a1"); + assertTrue(appsInA1.isEmpty()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.isEmpty()); + + rm.stop(); + } + + @Test + public void testMoveAllAppsInvalidDestination() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + List appsInB1 = scheduler.getAppsInQueue("b1"); + assertTrue(appsInB1.isEmpty()); + + List appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.isEmpty()); + + // now move the app + try { + scheduler.moveAllApps("a1", "DOES_NOT_EXIST"); + Assert.fail(); + } catch (YarnException e) { + // expected + } + + // check postconditions, app should still be in a1 + appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + appsInB1 = scheduler.getAppsInQueue("b1"); + assertTrue(appsInB1.isEmpty()); + + appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.isEmpty()); + + rm.stop(); + } + + @Test + public void testMoveAllAppsInvalidSource() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + List appsInB1 = scheduler.getAppsInQueue("b1"); + assertTrue(appsInB1.isEmpty()); + + List appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.isEmpty()); + + // now move the app + try { + scheduler.moveAllApps("DOES_NOT_EXIST", "b1"); + Assert.fail(); + } catch (YarnException e) { + // expected + } + + // check postconditions, app should still be in a1 + appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + appsInB1 = scheduler.getAppsInQueue("b1"); + assertTrue(appsInB1.isEmpty()); + + appsInB = scheduler.getAppsInQueue("b"); + assertTrue(appsInB.isEmpty()); + + rm.stop(); + } + } From e932365d6d46b5be16d0e79b751fac0b0b661400 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 15 Aug 2014 15:53:28 +0000 Subject: [PATCH 49/67] HADOOP-10698. KMS, add proxyuser support. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618217 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../crypto/key/kms/KMSClientProvider.java | 62 +++++-- .../kms/server/KMSAuthenticationFilter.java | 11 ++ .../hadoop-kms/src/site/apt/index.apt.vm | 40 +++++ .../hadoop/crypto/key/kms/server/TestKMS.java | 165 ++++++++++++++---- 5 files changed, 230 insertions(+), 50 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index aec02b4f61d..e18ad5fa28c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -210,6 +210,8 @@ Trunk (Unreleased) HADOOP-10770. KMS add delegation token support. (tucu) + HADOOP-10698. KMS, add proxyuser support. (tucu) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index 35439f26f6a..bce1eb5dd3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.ProviderUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.ssl.SSLFactory; @@ -52,6 +53,7 @@ import java.net.URL; import java.net.URLEncoder; import java.security.GeneralSecurityException; import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Date; @@ -235,6 +237,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, private SSLFactory sslFactory; private ConnectionConfigurator configurator; private DelegationTokenAuthenticatedURL.Token authToken; + private UserGroupInformation loginUgi; @Override public String toString() { @@ -316,6 +319,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT), new EncryptedQueueRefiller()); authToken = new DelegationTokenAuthenticatedURL.Token(); + loginUgi = UserGroupInformation.getCurrentUser(); } private String createServiceURL(URL url) throws IOException { @@ -374,14 +378,29 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, return conn; } - private HttpURLConnection createConnection(URL url, String method) + private HttpURLConnection createConnection(final URL url, String method) throws IOException { HttpURLConnection conn; try { - DelegationTokenAuthenticatedURL authUrl = - new DelegationTokenAuthenticatedURL(configurator); - conn = authUrl.openConnection(url, authToken); - } catch (AuthenticationException ex) { + // if current UGI is different from UGI at constructor time, behave as + // proxyuser + UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser(); + final String doAsUser = + (loginUgi.getShortUserName().equals(currentUgi.getShortUserName())) + ? null : currentUgi.getShortUserName(); + + // creating the HTTP connection using the current UGI at constructor time + conn = loginUgi.doAs(new PrivilegedExceptionAction() { + @Override + public HttpURLConnection run() throws Exception { + DelegationTokenAuthenticatedURL authUrl = + new DelegationTokenAuthenticatedURL(configurator); + return authUrl.openConnection(url, authToken, doAsUser); + } + }); + } catch (IOException ex) { + throw ex; + } catch (Exception ex) { throw new IOException(ex); } conn.setUseCaches(false); @@ -412,20 +431,27 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, if (status != expected) { InputStream es = null; try { - es = conn.getErrorStream(); - ObjectMapper mapper = new ObjectMapper(); - Map json = mapper.readValue(es, Map.class); - String exClass = (String) json.get( - KMSRESTConstants.ERROR_EXCEPTION_JSON); - String exMsg = (String) - json.get(KMSRESTConstants.ERROR_MESSAGE_JSON); Exception toThrow; - try { - ClassLoader cl = KMSClientProvider.class.getClassLoader(); - Class klass = cl.loadClass(exClass); - Constructor constr = klass.getConstructor(String.class); - toThrow = (Exception) constr.newInstance(exMsg); - } catch (Exception ex) { + String contentType = conn.getHeaderField(CONTENT_TYPE); + if (contentType != null && + contentType.toLowerCase().startsWith(APPLICATION_JSON_MIME)) { + es = conn.getErrorStream(); + ObjectMapper mapper = new ObjectMapper(); + Map json = mapper.readValue(es, Map.class); + String exClass = (String) json.get( + KMSRESTConstants.ERROR_EXCEPTION_JSON); + String exMsg = (String) + json.get(KMSRESTConstants.ERROR_MESSAGE_JSON); + try { + ClassLoader cl = KMSClientProvider.class.getClassLoader(); + Class klass = cl.loadClass(exClass); + Constructor constr = klass.getConstructor(String.class); + toThrow = (Exception) constr.newInstance(exMsg); + } catch (Exception ex) { + toThrow = new IOException(MessageFormat.format( + "HTTP status [{0}], {1}", status, conn.getResponseMessage())); + } + } else { toThrow = new IOException(MessageFormat.format( "HTTP status [{0}], {1}", status, conn.getResponseMessage())); } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 76c22c4646d..4df6db54084 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -75,6 +75,17 @@ public class KMSAuthenticationFilter return props; } + protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) { + Map proxyuserConf = KMSWebApp.getConfiguration(). + getValByRegex("hadoop\\.kms\\.proxyuser\\."); + Configuration conf = new Configuration(false); + for (Map.Entry entry : proxyuserConf.entrySet()) { + conf.set(entry.getKey().substring("hadoop.kms.".length()), + entry.getValue()); + } + return conf; + } + private static class KMSResponse extends HttpServletResponseWrapper { public int statusCode; public String msg; diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm index 0a1cba52e13..e0cbd780fd5 100644 --- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm +++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm @@ -195,6 +195,46 @@ hadoop-${project.version} $ sbin/kms.sh start NOTE: You need to restart the KMS for the configuration changes to take effect. +*** KMS Proxyuser Configuration + + Each proxyusers must be configured in <<>> using the + following properties: + ++---+ + + hadoop.kms.proxyusers.#USER#.users + * + + + + hadoop.kms.proxyusers.#USER#.groups + * + + + + hadoop.kms.proxyusers.#USER#.hosts + * + ++---+ + + <<<#USER#>>> is the username of the proxyuser to configure. + + The <<>> property indicates the users that can be impersonated. + + The <<>> property indicates the groups users being impersonated must + belong to. + + At least one of the <<>> or <<>> properties must be defined. + If both are specified, then the configured proxyuser will be able to + impersonate and user in the <<>> list and any user belonging to one of + the groups in the <<>> list. + + The <<>> property indicates from which host the proxyuser can make + impersonation requests. + + If <<>>, <<>> or <<>> has a <<<*>>>, it means there are + no restrictions for the proxyuser regarding users, groups or hosts. + *** KMS over HTTPS (SSL) To configure KMS to work over HTTPS the following 2 properties must be diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 971c6179b2c..be0a229b8d7 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -33,6 +33,7 @@ import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mortbay.jetty.Connector; @@ -71,6 +72,13 @@ import java.util.concurrent.Callable; public class TestKMS { + @Before + public void cleanUp() { + // resetting kerberos security + Configuration conf = new Configuration(); + UserGroupInformation.setConfiguration(conf); + } + public static File getTestDir() throws Exception { File file = new File("dummy"); file = file.getAbsoluteFile(); @@ -261,6 +269,7 @@ public class TestKMS { principals.add("HTTP/localhost"); principals.add("client"); principals.add("client/host"); + principals.add("client1"); for (KMSACLs.Type type : KMSACLs.Type.values()) { principals.add(type.toString()); } @@ -290,7 +299,9 @@ public class TestKMS { try { loginContext.login(); subject = loginContext.getSubject(); - return Subject.doAs(subject, action); + UserGroupInformation ugi = + UserGroupInformation.getUGIFromSubject(subject); + return ugi.doAs(action); } finally { loginContext.logout(); } @@ -298,8 +309,13 @@ public class TestKMS { public void testStartStop(final boolean ssl, final boolean kerberos) throws Exception { + Configuration conf = new Configuration(); + if (kerberos) { + conf.set("hadoop.security.authentication", "kerberos"); + } + UserGroupInformation.setConfiguration(conf); File testDir = getTestDir(); - Configuration conf = createBaseKMSConf(testDir); + conf = createBaseKMSConf(testDir); final String keystore; final String password; @@ -327,18 +343,18 @@ public class TestKMS { runServer(keystore, password, testDir, new KMSCallable() { @Override public Void call() throws Exception { - Configuration conf = new Configuration(); + final Configuration conf = new Configuration(); URL url = getKMSUrl(); Assert.assertEquals(keystore != null, url.getProtocol().equals("https")); - URI uri = createKMSUri(getKMSUrl()); - final KeyProvider kp = new KMSClientProvider(uri, conf); + final URI uri = createKMSUri(getKMSUrl()); if (kerberos) { for (String user : new String[]{"client", "client/host"}) { doAs(user, new PrivilegedExceptionAction() { @Override public Void run() throws Exception { + final KeyProvider kp = new KMSClientProvider(uri, conf); // getKeys() empty Assert.assertTrue(kp.getKeys().isEmpty()); return null; @@ -346,6 +362,7 @@ public class TestKMS { }); } } else { + KeyProvider kp = new KMSClientProvider(uri, conf); // getKeys() empty Assert.assertTrue(kp.getKeys().isEmpty()); } @@ -376,8 +393,11 @@ public class TestKMS { @Test public void testKMSProvider() throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(conf); File confDir = getTestDir(); - Configuration conf = createBaseKMSConf(confDir); + conf = createBaseKMSConf(confDir); writeConf(confDir, conf); runServer(null, null, confDir, new KMSCallable() { @@ -589,8 +609,11 @@ public class TestKMS { @Test public void testACLs() throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(conf); final File testDir = getTestDir(); - Configuration conf = createBaseKMSConf(testDir); + conf = createBaseKMSConf(testDir); conf.set("hadoop.kms.authentication.type", "kerberos"); conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath()); @@ -626,7 +649,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.createKey("k", new byte[16], new KeyProvider.Options(conf)); @@ -634,7 +657,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.rollNewVersion("k"); @@ -642,7 +665,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.rollNewVersion("k", new byte[16]); @@ -650,7 +673,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.getKeys(); @@ -658,7 +681,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.getKeysMetadata("k"); @@ -666,7 +689,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { // we are using JavaKeyStoreProvider for testing, so we know how @@ -676,7 +699,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.getCurrentKey("k"); @@ -684,7 +707,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.getMetadata("k"); @@ -692,7 +715,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } try { kp.getKeyVersions("k"); @@ -700,7 +723,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; @@ -716,7 +739,7 @@ public class TestKMS { new KeyProvider.Options(conf)); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -729,7 +752,7 @@ public class TestKMS { try { kp.deleteKey("k0"); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -744,7 +767,7 @@ public class TestKMS { new KeyProvider.Options(conf)); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -758,7 +781,7 @@ public class TestKMS { KeyProvider.KeyVersion kv = kp.rollNewVersion("k1"); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -773,7 +796,7 @@ public class TestKMS { kp.rollNewVersion("k1", new byte[16]); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -823,7 +846,7 @@ public class TestKMS { createKeyProviderCryptoExtension(kp); kpCE.decryptEncryptedKey(encKv); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -836,7 +859,7 @@ public class TestKMS { try { kp.getKeys(); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -850,7 +873,7 @@ public class TestKMS { kp.getMetadata("k1"); kp.getKeysMetadata("k1"); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -879,7 +902,7 @@ public class TestKMS { } catch (AuthorizationException ex) { //NOP } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; @@ -893,8 +916,11 @@ public class TestKMS { @Test public void testServicePrincipalACLs() throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(conf); File testDir = getTestDir(); - Configuration conf = createBaseKMSConf(testDir); + conf = createBaseKMSConf(testDir); conf.set("hadoop.kms.authentication.type", "kerberos"); conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath()); @@ -912,18 +938,19 @@ public class TestKMS { public Void call() throws Exception { final Configuration conf = new Configuration(); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128); - URI uri = createKMSUri(getKMSUrl()); - final KeyProvider kp = new KMSClientProvider(uri, conf); + conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 64); + final URI uri = createKMSUri(getKMSUrl()); doAs("client", new PrivilegedExceptionAction() { @Override public Void run() throws Exception { try { + KeyProvider kp = new KMSClientProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("ck0", new KeyProvider.Options(conf)); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -933,11 +960,12 @@ public class TestKMS { @Override public Void run() throws Exception { try { + KeyProvider kp = new KMSClientProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("ck1", new KeyProvider.Options(conf)); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { - Assert.fail(ex.toString()); + Assert.fail(ex.getMessage()); } return null; } @@ -1014,8 +1042,11 @@ public class TestKMS { @Test public void testDelegationTokenAccess() throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(conf); final File testDir = getTestDir(); - Configuration conf = createBaseKMSConf(testDir); + conf = createBaseKMSConf(testDir); conf.set("hadoop.kms.authentication.type", "kerberos"); conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath()); @@ -1076,4 +1107,74 @@ public class TestKMS { }); } + @Test + public void testProxyUser() throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(conf); + final File testDir = getTestDir(); + conf = createBaseKMSConf(testDir); + conf.set("hadoop.kms.authentication.type", "kerberos"); + conf.set("hadoop.kms.authentication.kerberos.keytab", + keytab.getAbsolutePath()); + conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost"); + conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT"); + conf.set("hadoop.kms.proxyuser.client.users", "foo"); + conf.set("hadoop.kms.proxyuser.client.hosts", "*"); + writeConf(testDir, conf); + + runServer(null, null, testDir, new KMSCallable() { + @Override + public Void call() throws Exception { + final Configuration conf = new Configuration(); + conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 64); + final URI uri = createKMSUri(getKMSUrl()); + + // proxyuser client using kerberos credentials + UserGroupInformation clientUgi = UserGroupInformation. + loginUserFromKeytabAndReturnUGI("client", keytab.getAbsolutePath()); + clientUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + final KeyProvider kp = new KMSClientProvider(uri, conf); + kp.createKey("kAA", new KeyProvider.Options(conf)); + + // authorized proxyuser + UserGroupInformation fooUgi = + UserGroupInformation.createRemoteUser("foo"); + fooUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + Assert.assertNotNull(kp.createKey("kBB", + new KeyProvider.Options(conf))); + return null; + } + }); + + // unauthorized proxyuser + UserGroupInformation foo1Ugi = + UserGroupInformation.createRemoteUser("foo1"); + foo1Ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + try { + kp.createKey("kCC", new KeyProvider.Options(conf)); + Assert.fail(); + } catch (AuthorizationException ex) { + // OK + } catch (Exception ex) { + Assert.fail(ex.getMessage()); + } + return null; + } + }); + return null; + } + }); + + return null; + } + }); + } + } From 31e478905af8cb4659bba5b0d3777791fb2945e7 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Fri, 15 Aug 2014 16:47:34 +0000 Subject: [PATCH 50/67] Wrong categorization for a variety of commits in the CHANGES.txt files git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618233 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 12 ++-- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +- hadoop-mapreduce-project/CHANGES.txt | 62 +++++++++---------- hadoop-yarn-project/CHANGES.txt | 12 ++-- 4 files changed, 46 insertions(+), 46 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e18ad5fa28c..c1e57c9621e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -202,12 +202,6 @@ Trunk (Unreleased) HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting underlying store. (asuresh via tucu) - HADOOP-8944. Shell command fs -count should include human readable option - (Jonathan Allen via aw) - - HADOOP-10231. Add some components in Native Libraries document (Akira - AJISAKA via aw) - HADOOP-10770. KMS add delegation token support. (tucu) HADOOP-10698. KMS, add proxyuser support. (tucu) @@ -521,6 +515,12 @@ Release 2.6.0 - UNRELEASED HADOOP-10281. Create a scheduler, which assigns schedulables a priority level. (Chris Li via Arpit Agarwal) + HADOOP-8944. Shell command fs -count should include human readable option + (Jonathan Allen via aw) + + HADOOP-10231. Add some components in Native Libraries document (Akira + AJISAKA via aw) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b47018a61b3..e7013146cd1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -137,9 +137,6 @@ Trunk (Unreleased) BUG FIXES - HDFS-6517. Remove hadoop-metrics2.properties from hdfs project (Akira - AJISAKA via aw) - HADOOP-9635 Fix potential Stack Overflow in DomainSocket.c (V. Karthik Kumar via cmccabe) @@ -414,6 +411,9 @@ Release 2.6.0 - UNRELEASED HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for insecure HDFS (Allen Wittenauer via raviprak) + HDFS-6517. Remove hadoop-metrics2.properties from hdfs project (Akira + AJISAKA via aw) + HDFS-6617. Flake TestDFSZKFailoverController.testManualFailoverWithDFSHAAdmin due to a long edit log sync op. (Liang Xie via cnauroth) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 05cce4ebde8..74a1fc163d6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -86,15 +86,6 @@ Trunk (Unreleased) MAPREDUCE-6019. MapReduce changes for exposing YARN/MR endpoints on multiple interfaces. (Craig Welch, Milan Potocnik, Arpit Agarwal via xgong) - MAPREDUCE-883. harchive: Document how to unarchive (Akira AJISAKA and - Koji Noguchi via aw) - - MAPREDUCE-4791. Javadoc for KeyValueTextInputFormat should include default - separator and how to change it (Akira AJISAKA via aw) - - MAPREDUCE-5906. Inconsistent configuration in property - "mapreduce.reduce.shuffle.input.buffer.percent" (Akira AJISAKA via aw) - BUG FIXES MAPREDUCE-5714. Removed forceful JVM exit in shutDownJob. @@ -160,28 +151,6 @@ Trunk (Unreleased) MAPREDUCE-5867. Fix NPE in KillAMPreemptionPolicy related to ProportionalCapacityPreemptionPolicy (Sunil G via devaraj) - MAPREDUCE-5944. Remove MRv1 commands from CommandsManual.apt.vm - (Akira AJISAKA via aw) - - MAPREDUCE-5943. Separate mapred commands from CommandManual.apt.vm - (Akira AJISAKA via aw) - - MAPREDUCE-5363. Fix doc and spelling for TaskCompletionEvent#getTaskStatus - and getStatus (Akira AJISAKA via aw) - - MAPREDUCE-5595. Typo in MergeManagerImpl.java (Akira AJISAKA via aw) - - MAPREDUCE-5597. Missing alternatives in javadocs for deprecated constructors - in mapreduce.Job (Akira AJISAKA via aw) - - MAPREDUCE-5950. incorrect description in distcp2 document (Akira AJISAKA - via aw) - - MAPREDUCE-5998. CompositeInputFormat javadoc is broken (Akira AJISAKA via - aw) - - MAPREDUCE-5999. Fix dead link in InputFormat javadoc (Akira AJISAKA via aw) - Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -196,6 +165,15 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5963. ShuffleHandler DB schema should be versioned with compatible/incompatible changes (Junping Du via jlowe) + MAPREDUCE-883. harchive: Document how to unarchive (Akira AJISAKA and + Koji Noguchi via aw) + + MAPREDUCE-4791. Javadoc for KeyValueTextInputFormat should include default + separator and how to change it (Akira AJISAKA via aw) + + MAPREDUCE-5906. Inconsistent configuration in property + "mapreduce.reduce.shuffle.input.buffer.percent" (Akira AJISAKA via aw) + OPTIMIZATIONS BUG FIXES @@ -224,6 +202,28 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5878. some standard JDK APIs are not part of system classes defaults (Sangjin Lee via jlowe) + MAPREDUCE-5944. Remove MRv1 commands from CommandsManual.apt.vm + (Akira AJISAKA via aw) + + MAPREDUCE-5943. Separate mapred commands from CommandManual.apt.vm + (Akira AJISAKA via aw) + + MAPREDUCE-5363. Fix doc and spelling for TaskCompletionEvent#getTaskStatus + and getStatus (Akira AJISAKA via aw) + + MAPREDUCE-5595. Typo in MergeManagerImpl.java (Akira AJISAKA via aw) + + MAPREDUCE-5597. Missing alternatives in javadocs for deprecated constructors + in mapreduce.Job (Akira AJISAKA via aw) + + MAPREDUCE-5950. incorrect description in distcp2 document (Akira AJISAKA + via aw) + + MAPREDUCE-5998. CompositeInputFormat javadoc is broken (Akira AJISAKA via + aw) + + MAPREDUCE-5999. Fix dead link in InputFormat javadoc (Akira AJISAKA via aw) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d0d32958f1e..495aae4746b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -8,12 +8,6 @@ Trunk - Unreleased IMPROVEMENTS - YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc - (Akira AJISAKA via aw) - - YARN-1918. Typo in description and error message for - 'yarn.resourcemanager.cluster-id' (Anandha L Ranganathan via aw) - OPTIMIZATIONS BUG FIXES @@ -55,6 +49,12 @@ Release 2.6.0 - UNRELEASED IMPROVEMENTS + YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc + (Akira AJISAKA via aw) + + YARN-1918. Typo in description and error message for + 'yarn.resourcemanager.cluster-id' (Anandha L Ranganathan via aw) + YARN-2242. Improve exception information on AM launch crashes. (Li Lu via junping_du) From 84bc2fe4021be32e0ff8ba395359337904149034 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Fri, 15 Aug 2014 20:17:45 +0000 Subject: [PATCH 51/67] =?UTF-8?q?MAPREDUCE-6032.=20Made=20MR=20jobs=20writ?= =?UTF-8?q?e=20job=20history=20files=20on=20the=20default=20FS=20when=20th?= =?UTF-8?q?e=20current=20context=E2=80=99s=20FS=20is=20different.=20Contri?= =?UTF-8?q?buted=20by=20Benjamin=20Zhitomirsky.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618269 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop-mapreduce-client-app/pom.xml | 6 ++ .../jobhistory/JobHistoryEventHandler.java | 12 ++- .../TestJobHistoryEventHandler.java | 97 ++++++++++++++++++- .../v2/jobhistory/JobHistoryUtils.java | 75 +++++++++++++- .../v2/hs/TestHistoryFileManager.java | 70 ++++++++++++- 6 files changed, 247 insertions(+), 16 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 74a1fc163d6..a48c6c18701 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -224,6 +224,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5999. Fix dead link in InputFormat javadoc (Akira AJISAKA via aw) + MAPREDUCE-6032. Made MR jobs write job history files on the default FS when + the current context’s FS is different. (Benjamin Zhitomirsky via zjshen) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index 4d1800f15c1..2e597d11d66 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -73,6 +73,12 @@ org.apache.hadoop hadoop-mapreduce-client-shuffle + + org.apache.hadoop + hadoop-hdfs + test-jar + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 8fd7b471600..c566740aa7a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -28,13 +28,13 @@ import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -74,7 +74,9 @@ public class JobHistoryEventHandler extends AbstractService private int eventCounter; - //TODO Does the FS object need to be different ? + // Those file systems may differ from the job configuration + // See org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils + // #ensurePathInDefaultFileSystem private FileSystem stagingDirFS; // log Dir FileSystem private FileSystem doneDirFS; // done Dir FileSystem @@ -141,7 +143,7 @@ public class JobHistoryEventHandler extends AbstractService //Check for the existence of the history staging dir. Maybe create it. try { stagingDirPath = - FileSystem.get(conf).makeQualified(new Path(stagingDirStr)); + FileContext.getFileContext(conf).makeQualified(new Path(stagingDirStr)); stagingDirFS = FileSystem.get(stagingDirPath.toUri(), conf); mkdir(stagingDirFS, stagingDirPath, new FsPermission( JobHistoryUtils.HISTORY_STAGING_DIR_PERMISSIONS)); @@ -154,7 +156,7 @@ public class JobHistoryEventHandler extends AbstractService //Check for the existence of intermediate done dir. Path doneDirPath = null; try { - doneDirPath = FileSystem.get(conf).makeQualified(new Path(doneDirStr)); + doneDirPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirStr)); doneDirFS = FileSystem.get(doneDirPath.toUri(), conf); // This directory will be in a common location, or this may be a cluster // meant for a single user. Creating based on the conf. Should ideally be @@ -194,7 +196,7 @@ public class JobHistoryEventHandler extends AbstractService //Check/create user directory under intermediate done dir. try { doneDirPrefixPath = - FileSystem.get(conf).makeQualified(new Path(userDoneDirStr)); + FileContext.getFileContext(conf).makeQualified(new Path(userDoneDirStr)); mkdir(doneDirFS, doneDirPrefixPath, new FsPermission( JobHistoryUtils.HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS)); } catch (IOException e) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index 167df3c8e29..d8a0cc7af52 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.never; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import org.junit.Assert; @@ -35,8 +36,13 @@ import org.junit.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -52,6 +58,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.junit.After; +import org.junit.AfterClass; +import static org.junit.Assert.assertFalse; +import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -60,6 +70,26 @@ public class TestJobHistoryEventHandler { private static final Log LOG = LogFactory .getLog(TestJobHistoryEventHandler.class); + private static MiniDFSCluster dfsCluster = null; + private static String coreSitePath; + + @BeforeClass + public static void setUpClass() throws Exception { + coreSitePath = "." + File.separator + "target" + File.separator + + "test-classes" + File.separator + "core-site.xml"; + Configuration conf = new HdfsConfiguration(); + dfsCluster = new MiniDFSCluster.Builder(conf).build(); + } + + @AfterClass + public static void cleanUpClass() throws Exception { + dfsCluster.shutdown(); + } + + @After + public void cleanTest() throws Exception { + new File(coreSitePath).delete(); + } @Test (timeout=50000) public void testFirstFlushOnCompletionEvent() throws Exception { @@ -325,6 +355,50 @@ public class TestJobHistoryEventHandler { } } + @Test (timeout=50000) + public void testDefaultFsIsUsedForHistory() throws Exception { + // Create default configuration pointing to the minicluster + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + dfsCluster.getURI().toString()); + FileOutputStream os = new FileOutputStream(coreSitePath); + conf.writeXml(os); + os.close(); + + // simulate execution under a non-default namenode + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + "file:///"); + + TestParams t = new TestParams(); + conf.set(MRJobConfig.MR_AM_STAGING_DIR, t.dfsWorkDir); + + JHEvenHandlerForTest realJheh = + new JHEvenHandlerForTest(t.mockAppContext, 0, false); + JHEvenHandlerForTest jheh = spy(realJheh); + jheh.init(conf); + + try { + jheh.start(); + handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( + t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000))); + + handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + new Counters(), new Counters()))); + + // If we got here then event handler worked but we don't know with which + // file system. Now we check that history stuff was written to minicluster + FileSystem dfsFileSystem = dfsCluster.getFileSystem(); + assertTrue("Minicluster contains some history files", + dfsFileSystem.globStatus(new Path(t.dfsWorkDir + "/*")).length != 0); + FileSystem localFileSystem = LocalFileSystem.get(conf); + assertFalse("No history directory on non-default file system", + localFileSystem.exists(new Path(t.dfsWorkDir))); + } finally { + jheh.stop(); + } + } + private void queueEvent(JHEvenHandlerForTest jheh, JobHistoryEvent event) { jheh.handle(event); } @@ -372,6 +446,7 @@ public class TestJobHistoryEventHandler { private class TestParams { boolean isLastAMRetry; String workDir = setupTestWorkDir(); + String dfsWorkDir = "/" + this.getClass().getCanonicalName(); ApplicationId appId = ApplicationId.newInstance(200, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -451,10 +526,16 @@ public class TestJobHistoryEventHandler { class JHEvenHandlerForTest extends JobHistoryEventHandler { private EventWriter eventWriter; + private boolean mockHistoryProcessing = true; public JHEvenHandlerForTest(AppContext context, int startCount) { super(context, startCount); } + public JHEvenHandlerForTest(AppContext context, int startCount, boolean mockHistoryProcessing) { + super(context, startCount); + this.mockHistoryProcessing = mockHistoryProcessing; + } + @Override protected void serviceStart() { } @@ -462,7 +543,12 @@ class JHEvenHandlerForTest extends JobHistoryEventHandler { @Override protected EventWriter createEventWriter(Path historyFilePath) throws IOException { - this.eventWriter = mock(EventWriter.class); + if (mockHistoryProcessing) { + this.eventWriter = mock(EventWriter.class); + } + else { + this.eventWriter = super.createEventWriter(historyFilePath); + } return this.eventWriter; } @@ -475,8 +561,13 @@ class JHEvenHandlerForTest extends JobHistoryEventHandler { } @Override - protected void processDoneFiles(JobId jobId){ - // do nothing + protected void processDoneFiles(JobId jobId) throws IOException { + if (!mockHistoryProcessing) { + super.processDoneFiles(jobId); + } + else { + // do nothing + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index d80fe40958d..167ee20a22e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -22,20 +22,24 @@ import java.io.File; import java.io.IOException; import java.util.Calendar; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -117,6 +121,7 @@ public class JobHistoryUtils { public static final String TIMESTAMP_DIR_REGEX = "\\d{4}" + "\\" + Path.SEPARATOR + "\\d{2}" + "\\" + Path.SEPARATOR + "\\d{2}"; public static final Pattern TIMESTAMP_DIR_PATTERN = Pattern.compile(TIMESTAMP_DIR_REGEX); private static final String TIMESTAMP_DIR_FORMAT = "%04d" + File.separator + "%02d" + File.separator + "%02d"; + private static final Log LOG = LogFactory.getLog(JobHistoryUtils.class); private static final PathFilter CONF_FILTER = new PathFilter() { @Override @@ -183,7 +188,7 @@ public class JobHistoryUtils { Path stagingPath = MRApps.getStagingAreaDir(conf, user); Path path = new Path(stagingPath, jobId); String logDir = path.toString(); - return logDir; + return ensurePathInDefaultFileSystem(logDir, conf); } /** @@ -200,7 +205,7 @@ public class JobHistoryUtils { MRJobConfig.DEFAULT_MR_AM_STAGING_DIR) + "/history/done_intermediate"; } - return doneDirPrefix; + return ensurePathInDefaultFileSystem(doneDirPrefix, conf); } /** @@ -216,7 +221,69 @@ public class JobHistoryUtils { MRJobConfig.DEFAULT_MR_AM_STAGING_DIR) + "/history/done"; } - return doneDirPrefix; + return ensurePathInDefaultFileSystem(doneDirPrefix, conf); + } + + /** + * Get default file system URI for the cluster (used to ensure consistency + * of history done/staging locations) over different context + * + * @return Default file context + */ + private static FileContext getDefaultFileContext() { + // If FS_DEFAULT_NAME_KEY was set solely by core-default.xml then we ignore + // ignore it. This prevents defaulting history paths to file system specified + // by core-default.xml which would not make sense in any case. For a test + // case to exploit this functionality it should create core-site.xml + FileContext fc = null; + Configuration defaultConf = new Configuration(); + String[] sources; + sources = defaultConf.getPropertySources( + CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); + if (sources != null && + (!Arrays.asList(sources).contains("core-default.xml") || + sources.length > 1)) { + try { + fc = FileContext.getFileContext(defaultConf); + LOG.info("Default file system [" + + fc.getDefaultFileSystem().getUri() + "]"); + } catch (UnsupportedFileSystemException e) { + LOG.error("Unable to create default file context [" + + defaultConf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) + + "]", + e); + } + } + else { + LOG.info("Default file system is set solely " + + "by core-default.xml therefore - ignoring"); + } + + return fc; + } + + /** + * Ensure that path belongs to cluster's default file system unless + * 1. it is already fully qualified. + * 2. current job configuration uses default file system + * 3. running from a test case without core-site.xml + * + * @param sourcePath source path + * @param conf the job configuration + * @return full qualified path (if necessary) in default file system + */ + private static String ensurePathInDefaultFileSystem(String sourcePath, Configuration conf) { + Path path = new Path(sourcePath); + FileContext fc = getDefaultFileContext(); + if (fc == null || + fc.getDefaultFileSystem().getUri().toString().equals( + conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "")) || + path.toUri().getAuthority() != null || + path.toUri().getScheme()!= null) { + return sourcePath; + } + + return fc.makeQualified(path).toString(); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index 34cb74b5a8b..1e07062d69c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -19,42 +19,74 @@ package org.apache.hadoop.mapreduce.v2.hs; +import java.io.File; +import java.io.FileOutputStream; +import java.util.UUID; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.mapreduce.v2.app.ControlledClock; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.test.CoreTestDriver; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; +import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; - -import java.util.UUID; +import org.junit.rules.TestName; public class TestHistoryFileManager { private static MiniDFSCluster dfsCluster = null; + private static MiniDFSCluster dfsCluster2 = null; + private static String coreSitePath; + + @Rule + public TestName name = new TestName(); @BeforeClass public static void setUpClass() throws Exception { + coreSitePath = "." + File.separator + "target" + File.separator + + "test-classes" + File.separator + "core-site.xml"; Configuration conf = new HdfsConfiguration(); + Configuration conf2 = new HdfsConfiguration(); dfsCluster = new MiniDFSCluster.Builder(conf).build(); + conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, + conf.get(MiniDFSCluster.HDFS_MINIDFS_BASEDIR) + "_2"); + dfsCluster2 = new MiniDFSCluster.Builder(conf2).build(); } @AfterClass public static void cleanUpClass() throws Exception { dfsCluster.shutdown(); + dfsCluster2.shutdown(); + } + + @After + public void cleanTest() throws Exception { + new File(coreSitePath).delete(); + } + + private String getDoneDirNameForTest() { + return "/" + name.getMethodName(); + } + + private String getIntermediateDoneDirNameForTest() { + return "/intermediate_" + name.getMethodName(); } private void testTryCreateHistoryDirs(Configuration conf, boolean expected) throws Exception { - conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, "/" + UUID.randomUUID()); - conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, "/" + UUID.randomUUID()); + conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, getDoneDirNameForTest()); + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, getIntermediateDoneDirNameForTest()); HistoryFileManager hfm = new HistoryFileManager(); hfm.conf = conf; Assert.assertEquals(expected, hfm.tryCreatingHistoryDirs(false)); @@ -75,6 +107,36 @@ public class TestHistoryFileManager { testTryCreateHistoryDirs(dfsCluster.getConfiguration(0), true); } + @Test + public void testCreateDirsWithAdditionalFileSystem() throws Exception { + dfsCluster.getFileSystem().setSafeMode( + HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + dfsCluster2.getFileSystem().setSafeMode( + HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + Assert.assertFalse(dfsCluster.getFileSystem().isInSafeMode()); + Assert.assertFalse(dfsCluster2.getFileSystem().isInSafeMode()); + + // Set default configuration to the first cluster + Configuration conf = new Configuration(false); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + dfsCluster.getURI().toString()); + FileOutputStream os = new FileOutputStream(coreSitePath); + conf.writeXml(os); + os.close(); + + testTryCreateHistoryDirs(dfsCluster2.getConfiguration(0), true); + + // Directories should be created only in the default file system (dfsCluster) + Assert.assertTrue(dfsCluster.getFileSystem() + .exists(new Path(getDoneDirNameForTest()))); + Assert.assertTrue(dfsCluster.getFileSystem() + .exists(new Path(getIntermediateDoneDirNameForTest()))); + Assert.assertFalse(dfsCluster2.getFileSystem() + .exists(new Path(getDoneDirNameForTest()))); + Assert.assertFalse(dfsCluster2.getFileSystem() + .exists(new Path(getIntermediateDoneDirNameForTest()))); + } + @Test public void testCreateDirsWithFileSystemInSafeMode() throws Exception { dfsCluster.getFileSystem().setSafeMode( From ddf108449d05e1b3c767d0dc81643b29b7d526b9 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Fri, 15 Aug 2014 20:25:51 +0000 Subject: [PATCH 52/67] MAPREDUCE-6032. Fix the non-unicode character in the change log in CHANGES.txt. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618270 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a48c6c18701..62b4e525644 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -225,7 +225,7 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5999. Fix dead link in InputFormat javadoc (Akira AJISAKA via aw) MAPREDUCE-6032. Made MR jobs write job history files on the default FS when - the current context’s FS is different. (Benjamin Zhitomirsky via zjshen) + the current context's FS is different. (Benjamin Zhitomirsky via zjshen) Release 2.5.0 - UNRELEASED From c3084d6c1661332a4771882529077e778d9c5692 Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 15 Aug 2014 23:53:57 +0000 Subject: [PATCH 53/67] YARN-2389. Added functionality for schedulers to kill all applications in a queue. Contributed by Subramaniam Venkatraman Krishnan git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618294 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/AbstractYarnScheduler.java | 21 ++++ .../scheduler/YarnScheduler.java | 8 ++ .../capacity/TestCapacityScheduler.java | 98 ++++++++++++++++++- 4 files changed, 127 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 495aae4746b..aa217edaec0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -140,6 +140,9 @@ Release 2.6.0 - UNRELEASED YARN-1370. Fair scheduler to re-populate container allocation state. (Anubhav Dhoot via kasha) + YARN-2389. Added functionality for schedulers to kill all applications in a + queue. (Subramaniam Venkatraman Krishnan via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index e5f22b8fc41..6463153c147 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -41,6 +41,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -348,4 +350,23 @@ public abstract class AbstractYarnScheduler .handle(new RMAppMoveEvent(app.getApplicationId(), destQueue, future)); } } + + @Override + public synchronized void killAllAppsInQueue(String queueName) + throws YarnException { + // check if queue is a valid + List apps = getAppsInQueue(queueName); + if (apps == null) { + String errMsg = "The specified Queue: " + queueName + " doesn't exist"; + LOG.warn(errMsg); + throw new YarnException(errMsg); + } + // generate kill events for each pending/running app + for (ApplicationAttemptId app : apps) { + this.rmContext + .getDispatcher() + .getEventHandler() + .handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL)); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 79dc773b410..5ce16c2b888 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -212,4 +212,12 @@ public interface YarnScheduler extends EventHandler { * @throws YarnException */ void moveAllApps(String sourceQueue, String destQueue) throws YarnException; + + /** + * Terminate all applications in the specified queue. + * + * @param queueName the name of queue to be drained + * @throws YarnException + */ + void killAllAppsInQueue(String queueName) throws YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 45c4950202d..f64bd62e078 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -1074,11 +1074,10 @@ public class TestCapacityScheduler { queue = scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue() .getQueueName(); - System.out.println(queue); Assert.assertTrue(queue.equals("b1")); appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInA.contains(appAttemptId)); + assertTrue(appsInB.contains(appAttemptId)); assertEquals(1, appsInB.size()); appsInRoot = scheduler.getAppsInQueue("root"); @@ -1140,6 +1139,7 @@ public class TestCapacityScheduler { assertTrue(appsInA1.isEmpty()); appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); appsInRoot = scheduler.getAppsInQueue("root"); @@ -1664,7 +1664,7 @@ public class TestCapacityScheduler { Assert.assertTrue(queue.equals("b1")); appsInB = scheduler.getAppsInQueue("b"); - assertTrue(appsInA.contains(appAttemptId)); + assertTrue(appsInB.contains(appAttemptId)); assertEquals(1, appsInB.size()); appsInRoot = scheduler.getAppsInQueue("root"); @@ -1798,4 +1798,96 @@ public class TestCapacityScheduler { rm.stop(); } + @Test + public void testKillAllAppsInQueue() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + String queue = + scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue() + .getQueueName(); + Assert.assertTrue(queue.equals("a1")); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + // now kill the app + scheduler.killAllAppsInQueue("a1"); + + // check postconditions + rm.waitForState(app.getApplicationId(), RMAppState.KILLED); + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.isEmpty()); + + appsInA1 = scheduler.getAppsInQueue("a1"); + assertTrue(appsInA1.isEmpty()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.isEmpty()); + + rm.stop(); + } + + @Test + public void testKillAllAppsInvalidSource() throws Exception { + MockRM rm = setUpMove(); + AbstractYarnScheduler scheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + + // submit an app + RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1"); + ApplicationAttemptId appAttemptId = + rm.getApplicationReport(app.getApplicationId()) + .getCurrentApplicationAttemptId(); + + // check preconditions + List appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + List appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + List appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + // now kill the app + try { + scheduler.killAllAppsInQueue("DOES_NOT_EXIST"); + Assert.fail(); + } catch (YarnException e) { + // expected + } + + // check postconditions, app should still be in a1 + appsInA1 = scheduler.getAppsInQueue("a1"); + assertEquals(1, appsInA1.size()); + + appsInA = scheduler.getAppsInQueue("a"); + assertTrue(appsInA.contains(appAttemptId)); + assertEquals(1, appsInA.size()); + + appsInRoot = scheduler.getAppsInQueue("root"); + assertTrue(appsInRoot.contains(appAttemptId)); + assertEquals(1, appsInRoot.size()); + + rm.stop(); + } + } From c1cd41cc49ad99e733dc6b4e90cf29bb8020be06 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 17 Aug 2014 17:06:01 +0000 Subject: [PATCH 54/67] HADOOP-10650. Add ability to specify a reverse ACL (black list) of users and groups. (Contributed by Benoy Antony) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618482 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/fs/CommonConfigurationKeys.java | 3 + .../ServiceAuthorizationManager.java | 38 ++++-- .../src/site/apt/ServiceLevelAuth.apt.vm | 21 ++++ .../authorize/TestServiceAuthorization.java | 117 ++++++++++++++++++ 5 files changed, 171 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c1e57c9621e..d0168c2d5bc 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -521,6 +521,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10231. Add some components in Native Libraries document (Akira AJISAKA via aw) + HADOOP-10650. Add ability to specify a reverse ACL (black list) of users + and groups. (Benoy Antony via Arpit Agarwal) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 3345e3c93d5..b4aedb37aef 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -134,6 +134,9 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL = "security.service.authorization.default.acl"; public static final String + HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_BLOCKED_ACL = + "security.service.authorization.default.acl.blocked"; + public static final String HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_POLICY = "security.refresh.policy.protocol.acl"; public static final String diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index d12ab79c798..272538a90fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -43,10 +43,14 @@ import com.google.common.annotations.VisibleForTesting; @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public class ServiceAuthorizationManager { + static final String BLOCKED = ".blocked"; + private static final String HADOOP_POLICY_FILE = "hadoop-policy.xml"; - private volatile Map, AccessControlList> protocolToAcl = - new IdentityHashMap, AccessControlList>(); + // For each class, first ACL in the array specifies the allowed entries + // and second ACL specifies blocked entries. + private volatile Map, AccessControlList[]> protocolToAcls = + new IdentityHashMap, AccessControlList[]>(); /** * Configuration key for controlling service-level authorization for Hadoop. @@ -80,8 +84,8 @@ public class ServiceAuthorizationManager { Configuration conf, InetAddress addr ) throws AuthorizationException { - AccessControlList acl = protocolToAcl.get(protocol); - if (acl == null) { + AccessControlList[] acls = protocolToAcls.get(protocol); + if (acls == null) { throw new AuthorizationException("Protocol " + protocol + " is not known."); } @@ -104,7 +108,7 @@ public class ServiceAuthorizationManager { } } if((clientPrincipal != null && !clientPrincipal.equals(user.getUserName())) || - !acl.isUserAllowed(user)) { + acls.length != 2 || !acls[0].isUserAllowed(user) || acls[1].isUserAllowed(user)) { AUDITLOG.warn(AUTHZ_FAILED_FOR + user + " for protocol=" + protocol + ", expected client Kerberos principal is " + clientPrincipal); throw new AuthorizationException("User " + user + @@ -129,13 +133,16 @@ public class ServiceAuthorizationManager { @Private public void refreshWithLoadedConfiguration(Configuration conf, PolicyProvider provider) { - final Map, AccessControlList> newAcls = - new IdentityHashMap, AccessControlList>(); + final Map, AccessControlList[]> newAcls = + new IdentityHashMap, AccessControlList[]>(); String defaultAcl = conf.get( CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL, AccessControlList.WILDCARD_ACL_VALUE); + String defaultBlockedAcl = conf.get( + CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_BLOCKED_ACL, ""); + // Parse the config file Service[] services = provider.getServices(); if (services != null) { @@ -145,21 +152,30 @@ public class ServiceAuthorizationManager { conf.get(service.getServiceKey(), defaultAcl) ); - newAcls.put(service.getProtocol(), acl); + AccessControlList blockedAcl = + new AccessControlList( + conf.get(service.getServiceKey() + BLOCKED, + defaultBlockedAcl)); + newAcls.put(service.getProtocol(), new AccessControlList[] {acl, blockedAcl}); } } // Flip to the newly parsed permissions - protocolToAcl = newAcls; + protocolToAcls = newAcls; } @VisibleForTesting public Set> getProtocolsWithAcls() { - return protocolToAcl.keySet(); + return protocolToAcls.keySet(); } @VisibleForTesting public AccessControlList getProtocolsAcls(Class className) { - return protocolToAcl.get(className); + return protocolToAcls.get(className)[0]; + } + + @VisibleForTesting + public AccessControlList getProtocolsBlockedAcls(Class className) { + return protocolToAcls.get(className)[1]; } } diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm index 6a11f3f643d..6f714545ffe 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm @@ -110,6 +110,27 @@ security.ha.service.protocol.acl | ACL for HAService protocol used by HAAdm <<>> is applied. If <<>> is not defined, <<<*>>> is applied. + ** Blocked Access Control Lists + + In some cases, it is required to specify blocked access control list for a service. This specifies + the list of users and groups who are not authorized to access the service. The format of + the blocked access control list is same as that of access control list. The blocked access + control list can be specified via <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>>. The property name + is derived by suffixing with ".blocked". + + Example: The property name of blocked access control list for <<> + will be <<>> + + For a service, it is possible to specify both an access control list and a blocked control + list. A user is authorized to access the service if the user is in the access control and not in + the blocked access control list. + + If blocked access control list is not defined for a service, the value of + <<>> is applied. If + <<>> is not defined, + empty blocked access control list is applied. + + ** Refreshing Service Level Authorization Configuration The service-level authorization configuration for the NameNode and diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java index f6cf8bce2e9..9ef9d7add53 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java @@ -18,16 +18,22 @@ package org.apache.hadoop.security.authorize; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.net.InetAddress; +import java.net.UnknownHostException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.ipc.TestRPC.TestProtocol; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.Test; public class TestServiceAuthorization { private static final String ACL_CONFIG = "test.protocol.acl"; private static final String ACL_CONFIG1 = "test.protocol1.acl"; + private static final String ADDRESS = "0.0.0.0"; public interface TestProtocol1 extends TestProtocol {}; @@ -64,4 +70,115 @@ public class TestServiceAuthorization { acl = serviceAuthorizationManager.getProtocolsAcls(TestProtocol1.class); assertEquals("user2 group2", acl.getAclString()); } + + @Test + public void testBlockedAcl() throws UnknownHostException { + UserGroupInformation drwho = + UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM", + new String[] { "group1", "group2" }); + + ServiceAuthorizationManager serviceAuthorizationManager = + new ServiceAuthorizationManager(); + Configuration conf = new Configuration (); + + //test without setting a blocked acl + conf.set(ACL_CONFIG, "user1 group1"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, + InetAddress.getByName(ADDRESS)); + } catch (AuthorizationException e) { + fail(); + } + //now set a blocked acl with another user and another group + conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho2 group3"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, + InetAddress.getByName(ADDRESS)); + } catch (AuthorizationException e) { + fail(); + } + //now set a blocked acl with the user and another group + conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho group3"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, + InetAddress.getByName(ADDRESS)); + fail(); + } catch (AuthorizationException e) { + + } + //now set a blocked acl with another user and another group + conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho2 group3"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, + InetAddress.getByName(ADDRESS)); + } catch (AuthorizationException e) { + fail(); + } + //now set a blocked acl with another user and group that the user belongs to + conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho2 group2"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, + InetAddress.getByName(ADDRESS)); + fail(); + } catch (AuthorizationException e) { + //expects Exception + } + //reset blocked acl so that there is no blocked ACL + conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, ""); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, + InetAddress.getByName(ADDRESS)); + } catch (AuthorizationException e) { + fail(); + } + } + + @Test + public void testDefaultBlockedAcl() throws UnknownHostException { + UserGroupInformation drwho = + UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM", + new String[] { "group1", "group2" }); + + ServiceAuthorizationManager serviceAuthorizationManager = + new ServiceAuthorizationManager(); + Configuration conf = new Configuration (); + + //test without setting a default blocked acl + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol1.class, conf, + InetAddress.getByName(ADDRESS)); + } catch (AuthorizationException e) { + fail(); + } + + //set a restrictive default blocked acl and an non-restricting blocked acl for TestProtocol + conf.set( + CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_BLOCKED_ACL, + "user2 group2"); + conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "user2"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + //drwho is authorized to access TestProtocol + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf, + InetAddress.getByName(ADDRESS)); + } catch (AuthorizationException e) { + fail(); + } + //drwho is not authorized to access TestProtocol1 because it uses the default blocked acl. + try { + serviceAuthorizationManager.authorize(drwho, TestProtocol1.class, conf, + InetAddress.getByName(ADDRESS)); + fail(); + } catch (AuthorizationException e) { + //expects Exception + } + } + } From b47ad1ccbaf6a75eecfbeddb17e539480a01aab3 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 17 Aug 2014 17:43:20 +0000 Subject: [PATCH 55/67] HADOOP-10335. An ip whilelist based implementation to resolve Sasl properties per connection. (Contributed by Benoy Antony) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618484 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../security/WhitelistBasedResolver.java | 149 ++++++++++++ .../apache/hadoop/util/CacheableIPList.java | 76 +++++++ .../hadoop/util/CombinedIPWhiteList.java | 60 +++++ .../apache/hadoop/util/FileBasedIPList.java | 102 +++++++++ .../java/org/apache/hadoop/util/IPList.java | 33 +++ .../org/apache/hadoop/util/MachineList.java | 13 +- .../security/TestWhitelistBasedResolver.java | 163 +++++++++++++ .../hadoop/util/TestCacheableIPList.java | 188 +++++++++++++++ .../hadoop/util/TestFileBasedIPList.java | 215 ++++++++++++++++++ 10 files changed, 999 insertions(+), 3 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d0168c2d5bc..fb0578f7c50 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -524,6 +524,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10650. Add ability to specify a reverse ACL (black list) of users and groups. (Benoy Antony via Arpit Agarwal) + HADOOP-10335. An ip whilelist based implementation to resolve Sasl + properties per connection. (Benoy Antony via Arpit Agarwal) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java new file mode 100644 index 00000000000..dc0815ed768 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; +import java.util.TreeMap; + +import javax.security.sasl.Sasl; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SaslPropertiesResolver; +import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; +import org.apache.hadoop.util.CombinedIPWhiteList; +import org.apache.hadoop.util.StringUtils; + + +/** + * An implementation of the SaslPropertiesResolver. + * Uses a white list of IPs. + * If the connection's IP address is in the list of IP addresses, the salProperties + * will be unchanged. + * If the connection's IP is not in the list of IP addresses, then QOP for the + * connection will be restricted to "hadoop.rpc.protection.non-whitelist" + * + * Uses 3 IPList implementations together to form an aggregate whitelist. + * 1. ConstantIPList - to check against a set of hardcoded IPs + * 2. Fixed IP List - to check against a list of IP addresses which are specified externally, but + * will not change over runtime. + * 3. Variable IP List - to check against a list of IP addresses which are specified externally and + * could change during runtime. + * A connection IP address will checked against these 3 IP Lists in the order specified above. + * Once a match is found , the IP address is determined to be in whitelist. + * + * The behavior can be configured using a bunch of configuration parameters. + * + */ +public class WhitelistBasedResolver extends SaslPropertiesResolver { + public static final Log LOG = LogFactory.getLog(WhitelistBasedResolver.class); + + private static final String FIXEDWHITELIST_DEFAULT_LOCATION = "/etc/hadoop/fixedwhitelist"; + + private static final String VARIABLEWHITELIST_DEFAULT_LOCATION = "/etc/hadoop/whitelist"; + + /** + * Path to the file to containing subnets and ip addresses to form fixed whitelist. + */ + public static final String HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE = + "hadoop.security.sasl.fixedwhitelist.file"; + /** + * Enables/Disables variable whitelist + */ + public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE = + "hadoop.security.sasl.variablewhitelist.enable"; + /** + * Path to the file to containing subnets and ip addresses to form variable whitelist. + */ + public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE = + "hadoop.security.sasl.variablewhitelist.file"; + /** + * time in seconds by which the variable whitelist file is checked for updates + */ + public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS = + "hadoop.security.sasl.variablewhitelist.cache.secs"; + + /** + * comma separated list containing alternate hadoop.rpc.protection values for + * clients which are not in whitelist + */ + public static final String HADOOP_RPC_PROTECTION_NON_WHITELIST = + "hadoop.rpc.protection.non-whitelist"; + + private CombinedIPWhiteList whiteList; + + private Map saslProps; + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + String fixedFile = conf.get(HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE, + FIXEDWHITELIST_DEFAULT_LOCATION); + String variableFile = null; + long expiryTime = 0; + + if (conf.getBoolean(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, false)) { + variableFile = conf.get(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE, + VARIABLEWHITELIST_DEFAULT_LOCATION); + expiryTime = + conf.getLong(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS,3600) * 1000; + } + + whiteList = new CombinedIPWhiteList(fixedFile,variableFile,expiryTime); + + this.saslProps = getSaslProperties(conf); + } + + /** + * Identify the Sasl Properties to be used for a connection with a client. + * @param clientAddress client's address + * @return the sasl properties to be used for the connection. + */ + @Override + public Map getServerProperties(InetAddress clientAddress) { + if (clientAddress == null) { + return saslProps; + } + return whiteList.isIn(clientAddress.getHostAddress())?getDefaultProperties():saslProps; + } + + public Map getServerProperties(String clientAddress) throws UnknownHostException { + if (clientAddress == null) { + return saslProps; + } + return getServerProperties(InetAddress.getByName(clientAddress)); + } + + static Map getSaslProperties(Configuration conf) { + Map saslProps =new TreeMap(); + String[] qop = conf.getStrings(HADOOP_RPC_PROTECTION_NON_WHITELIST, + QualityOfProtection.PRIVACY.toString()); + + for (int i=0; i < qop.length; i++) { + qop[i] = QualityOfProtection.valueOf(qop[i].toUpperCase()).getSaslQop(); + } + + saslProps.put(Sasl.QOP, StringUtils.join(",", qop)); + saslProps.put(Sasl.SERVER_AUTH, "true"); + + return saslProps; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java new file mode 100644 index 00000000000..1343fc6129e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +/** + * CacheableIPList loads a list of subnets from a file. + * The list is cached and the cache can be refreshed by specifying cache timeout. + * A negative value of cache timeout disables any caching. + * + * Thread safe. + */ + +public class CacheableIPList implements IPList { + private final long cacheTimeout; + private volatile long cacheExpiryTimeStamp; + private volatile FileBasedIPList ipList; + + public CacheableIPList(FileBasedIPList ipList, long cacheTimeout) { + this.cacheTimeout = cacheTimeout; + this.ipList = ipList; + updateCacheExpiryTime(); + } + + /** + * Reloads the ip list + */ + private void reset() { + ipList = ipList.reload(); + updateCacheExpiryTime(); + } + + private void updateCacheExpiryTime() { + if (cacheTimeout < 0) { + cacheExpiryTimeStamp = -1; // no automatic cache expiry. + }else { + cacheExpiryTimeStamp = System.currentTimeMillis() + cacheTimeout; + } + } + + /** + * Refreshes the ip list + */ + public void refresh () { + cacheExpiryTimeStamp = 0; + } + + @Override + public boolean isIn(String ipAddress) { + //is cache expired + //Uses Double Checked Locking using volatile + if (cacheExpiryTimeStamp >= 0 && cacheExpiryTimeStamp < System.currentTimeMillis()) { + synchronized(this) { + //check if cache expired again + if (cacheExpiryTimeStamp < System.currentTimeMillis()) { + reset(); + } + } + } + return ipList.isIn(ipAddress); + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java new file mode 100644 index 00000000000..d12c4c11d5d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class CombinedIPWhiteList implements IPList { + + public static final Log LOG = LogFactory.getLog(CombinedIPWhiteList.class); + private static final String LOCALHOST_IP = "127.0.0.1"; + + private final IPList[] networkLists; + + public CombinedIPWhiteList(String fixedWhiteListFile, + String variableWhiteListFile, long cacheExpiryInSeconds) { + + IPList fixedNetworkList = new FileBasedIPList(fixedWhiteListFile); + if (variableWhiteListFile != null){ + IPList variableNetworkList = new CacheableIPList( + new FileBasedIPList(variableWhiteListFile),cacheExpiryInSeconds); + networkLists = new IPList[] {fixedNetworkList, variableNetworkList}; + } + else { + networkLists = new IPList[] {fixedNetworkList}; + } + } + @Override + public boolean isIn(String ipAddress) { + if (ipAddress == null) { + throw new IllegalArgumentException("ipAddress is null"); + } + + if (LOCALHOST_IP.equals(ipAddress)) { + return true; + } + + for (IPList networkList:networkLists) { + if (networkList.isIn(ipAddress)) { + return true; + } + } + return false; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java new file mode 100644 index 00000000000..8bfb5d93aef --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * FileBasedIPList loads a list of subnets in CIDR format and ip addresses from a file. + * + * Given an ip address, isIn method returns true if ip belongs to one of the subnets. + * + * Thread safe. + */ + +public class FileBasedIPList implements IPList { + + private static final Log LOG = LogFactory.getLog(FileBasedIPList.class); + + private final String fileName; + private final MachineList addressList; + + public FileBasedIPList(String fileName) { + this.fileName = fileName; + String[] lines = readLines(fileName); + if (lines != null) { + addressList = new MachineList(new HashSet(Arrays.asList(lines))); + } else { + addressList = null; + } + } + + public FileBasedIPList reload() { + return new FileBasedIPList(fileName); + } + + @Override + public boolean isIn(String ipAddress) { + if (ipAddress == null || addressList == null) { + return false; + } + return addressList.includes(ipAddress); + } + + /** + * reads the lines in a file. + * @param fileName + * @return lines in a String array; null if the file does not exist or if the + * file name is null + * @throws IOException + */ + private static String[] readLines(String fileName) { + try { + if (fileName != null) { + File file = new File (fileName); + if (file.exists()) { + FileReader fileReader = new FileReader(file); + BufferedReader bufferedReader = new BufferedReader(fileReader); + List lines = new ArrayList(); + String line = null; + while ((line = bufferedReader.readLine()) != null) { + lines.add(line); + } + bufferedReader.close(); + LOG.debug("Loaded IP list of size = " + lines.size() +" from file = " + fileName); + return(lines.toArray(new String[lines.size()])); + } + else { + LOG.debug("Missing ip list file : "+ fileName); + } + } + } + catch (Throwable t) { + LOG.error(t); + } + return null; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java new file mode 100644 index 00000000000..3a2616376fb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceStability.Unstable +@InterfaceAudience.Public +public interface IPList { + + /** + * returns true if the ipAddress is in the IPList. + * @param ipAddress + * @return boolean value indicating whether the ipAddress is in the IPList + */ + public abstract boolean isIn(String ipAddress); +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java index 7f2e2c8de7e..d1a0870f679 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java @@ -37,7 +37,7 @@ import com.google.common.net.InetAddresses; /** * Container class which holds a list of ip/host addresses and * answers membership queries. - * . + * * Accepts list of ip addresses, ip addreses in CIDR format and/or * host addresses. */ @@ -71,8 +71,15 @@ public class MachineList { * @param hostEntries comma separated ip/cidr/host addresses */ public MachineList(String hostEntries) { - this(StringUtils.getTrimmedStringCollection(hostEntries), - InetAddressFactory.S_INSTANCE); + this(StringUtils.getTrimmedStringCollection(hostEntries)); + } + + /** + * + * @param hostEntries collection of separated ip/cidr/host addresses + */ + public MachineList(Collection hostEntries) { + this(hostEntries, InetAddressFactory.S_INSTANCE); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java new file mode 100644 index 00000000000..684ef3bf17a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.WhitelistBasedResolver; +import org.apache.hadoop.util.TestFileBasedIPList; + +public class TestWhitelistBasedResolver extends TestCase { + + public static final Map SASL_PRIVACY_PROPS = + WhitelistBasedResolver.getSaslProperties(new Configuration()); + + public void testFixedVariableAndLocalWhiteList() throws IOException { + + String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; + + TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); + + String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); + + Configuration conf = new Configuration(); + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , + "fixedwhitelist.txt"); + + conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, + true); + + conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, + 1); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , + "variablewhitelist.txt"); + + WhitelistBasedResolver wqr = new WhitelistBasedResolver (); + wqr.setConf(conf); + + assertEquals (wqr.getDefaultProperties(), + wqr.getServerProperties(InetAddress.getByName("10.119.103.112"))); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.222.103.121")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.113.221.221")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1")); + + TestFileBasedIPList.removeFile("fixedwhitelist.txt"); + TestFileBasedIPList.removeFile("variablewhitelist.txt"); + } + + + /** + * Add a bunch of subnets and IPSs to the whitelist + * Check for inclusion in whitelist + * Check for exclusion from whitelist + */ + public void testFixedAndLocalWhiteList() throws IOException { + + String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; + + TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); + + String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); + + Configuration conf = new Configuration(); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , + "fixedwhitelist.txt"); + + conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, + false); + + conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, + 100); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , + "variablewhitelist.txt"); + + WhitelistBasedResolver wqr = new WhitelistBasedResolver(); + wqr.setConf(conf); + + assertEquals (wqr.getDefaultProperties(), + wqr.getServerProperties(InetAddress.getByName("10.119.103.112"))); + + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113")); + + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121")); + + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.222.103.121")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.221")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1"));; + + TestFileBasedIPList.removeFile("fixedwhitelist.txt"); + TestFileBasedIPList.removeFile("variablewhitelist.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the whitelist + * Check for inclusion in whitelist with a null value + */ + public void testNullIPAddress() throws IOException { + + String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; + + TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); + + String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); + + Configuration conf = new Configuration(); + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , + "fixedwhitelist.txt"); + + conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, + true); + + conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, + 100); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , + "variablewhitelist.txt"); + + WhitelistBasedResolver wqr = new WhitelistBasedResolver(); + wqr.setConf(conf); + + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties((InetAddress)null)); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties((String)null)); + + TestFileBasedIPList.removeFile("fixedwhitelist.txt"); + TestFileBasedIPList.removeFile("variablewhitelist.txt"); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java new file mode 100644 index 00000000000..3289d786c13 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.io.IOException; + +import org.apache.hadoop.util.CacheableIPList; +import org.apache.hadoop.util.FileBasedIPList; + + +import junit.framework.TestCase; + +public class TestCacheableIPList extends TestCase { + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Add a bunch of subnets and Ips + * wait for cache timeout. + * test for inclusion + * Check for exclusion + */ + public void testAddWithSleepForCacheTimeout() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + Thread.sleep(101); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Remove a bunch of subnets and Ips + * wait for cache timeout. + * test for inclusion + * Check for exclusion + */ + public void testRemovalWithSleepForCacheTimeout() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + Thread.sleep(1005); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Add a bunch of subnets and Ips + * do a refresh + * test for inclusion + * Check for exclusion + */ + public void testAddWithRefresh() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + cipl.refresh(); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Remove a bunch of subnets and Ips + * wait for cache timeout. + * test for inclusion + * Check for exclusion + */ + public void testRemovalWithRefresh() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + cipl.refresh(); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java new file mode 100644 index 00000000000..0e79fd1bbab --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.util.FileBasedIPList; +import org.apache.hadoop.util.IPList; +import org.junit.After; +import org.junit.Test; + +import junit.framework.TestCase; + +public class TestFileBasedIPList extends TestCase { + + @After + public void tearDown() { + removeFile("ips.txt"); + } + + /** + * Add a bunch of IPS to the file + * Check for inclusion + * Check for exclusion + */ + @Test + public void testSubnetsAndIPs() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23"}; + + createFileWithEntries ("ips.txt", ips); + + IPList ipList = new FileBasedIPList("ips.txt"); + + assertTrue ("10.119.103.112 is not in the list", + ipList.isIn("10.119.103.112")); + assertFalse ("10.119.103.113 is in the list", + ipList.isIn("10.119.103.113")); + + assertTrue ("10.221.102.0 is not in the list", + ipList.isIn("10.221.102.0")); + assertTrue ("10.221.102.1 is not in the list", + ipList.isIn("10.221.102.1")); + assertTrue ("10.221.103.1 is not in the list", + ipList.isIn("10.221.103.1")); + assertTrue ("10.221.103.255 is not in the list", + ipList.isIn("10.221.103.255")); + assertFalse("10.221.104.0 is in the list", + ipList.isIn("10.221.104.0")); + assertFalse("10.221.104.1 is in the list", + ipList.isIn("10.221.104.1")); + } + + /** + * Add a bunch of IPS to the file + * Check for inclusion + * Check for exclusion + */ + @Test + public void testNullIP() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23"}; + createFileWithEntries ("ips.txt", ips); + + IPList ipList = new FileBasedIPList("ips.txt"); + + assertFalse ("Null Ip is in the list", + ipList.isIn(null)); + } + + /** + * Add a bunch of subnets and IPSs to the file + * Check for inclusion + * Check for exclusion + */ + @Test + public void testWithMultipleSubnetAndIPs() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.222.0.0/16", + "10.113.221.221"}; + + createFileWithEntries ("ips.txt", ips); + + IPList ipList = new FileBasedIPList("ips.txt"); + + assertTrue ("10.119.103.112 is not in the list", + ipList.isIn("10.119.103.112")); + assertFalse ("10.119.103.113 is in the list", + ipList.isIn("10.119.103.113")); + + assertTrue ("10.221.103.121 is not in the list", + ipList.isIn("10.221.103.121")); + assertFalse("10.221.104.0 is in the list", + ipList.isIn("10.221.104.0")); + + assertTrue ("10.222.103.121 is not in the list", + ipList.isIn("10.222.103.121")); + assertFalse("10.223.104.0 is in the list", + ipList.isIn("10.223.104.0")); + + assertTrue ("10.113.221.221 is not in the list", + ipList.isIn("10.113.221.221")); + assertFalse("10.113.221.222 is in the list", + ipList.isIn("10.113.221.222")); + } + + /** + * Do not specify the file + * test for inclusion + * should be true as if the feature is turned off + */ + public void testFileNotSpecified() { + + IPList ipl = new FileBasedIPList(null); + + assertFalse("110.113.221.222 is in the list", + ipl.isIn("110.113.221.222")); + } + + /** + * Specify a non existent file + * test for inclusion + * should be true as if the feature is turned off + */ + public void testFileMissing() { + + IPList ipl = new FileBasedIPList("missingips.txt"); + + assertFalse("110.113.221.222 is in the list", + ipl.isIn("110.113.221.222")); + } + + /** + * Specify an existing file, but empty + * test for inclusion + * should be true as if the feature is turned off + */ + public void testWithEmptyList() throws IOException { + String[] ips = {}; + + createFileWithEntries ("ips.txt", ips); + IPList ipl = new FileBasedIPList("ips.txt"); + + assertFalse("110.113.221.222 is in the list", + ipl.isIn("110.113.221.222")); + } + + /** + * Specify an existing file, but ips in wrong format + * test for inclusion + * should be true as if the feature is turned off + */ + public void testForBadFIle() throws IOException { + String[] ips = { "10.221.102/23"}; + + createFileWithEntries ("ips.txt", ips); + + try { + new FileBasedIPList("ips.txt"); + fail(); + } catch (Exception e) { + //expects Exception + } + } + + /** + * Add a bunch of subnets and IPSs to the file. Keep one entry wrong. + * The good entries will still be used. + * Check for inclusion with good entries + * Check for exclusion + */ + public void testWithAWrongEntry() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102/23", "10.221.204.1/23"}; + + createFileWithEntries ("ips.txt", ips); + + try { + new FileBasedIPList("ips.txt"); + fail(); + } catch (Exception e) { + //expects Exception + } + } + + public static void createFileWithEntries(String fileName, String[] ips) + throws IOException { + FileUtils.writeLines(new File(fileName), Arrays.asList(ips)); + } + + public static void removeFile(String fileName) { + File file = new File(fileName); + if (file.exists()) { + new File(fileName).delete(); + } + } +} From 48505679c03f953fd045cc8077e210ef191c51dd Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 17 Aug 2014 17:47:18 +0000 Subject: [PATCH 56/67] HADOOP-10335. Undo checkin to resolve test build issue. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618487 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 - .../security/WhitelistBasedResolver.java | 149 ------------ .../apache/hadoop/util/CacheableIPList.java | 76 ------- .../hadoop/util/CombinedIPWhiteList.java | 60 ----- .../apache/hadoop/util/FileBasedIPList.java | 102 --------- .../java/org/apache/hadoop/util/IPList.java | 33 --- .../org/apache/hadoop/util/MachineList.java | 13 +- .../security/TestWhitelistBasedResolver.java | 163 ------------- .../hadoop/util/TestCacheableIPList.java | 188 --------------- .../hadoop/util/TestFileBasedIPList.java | 215 ------------------ 10 files changed, 3 insertions(+), 999 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fb0578f7c50..d0168c2d5bc 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -524,9 +524,6 @@ Release 2.6.0 - UNRELEASED HADOOP-10650. Add ability to specify a reverse ACL (black list) of users and groups. (Benoy Antony via Arpit Agarwal) - HADOOP-10335. An ip whilelist based implementation to resolve Sasl - properties per connection. (Benoy Antony via Arpit Agarwal) - OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java deleted file mode 100644 index dc0815ed768..00000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.security; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Map; -import java.util.TreeMap; - -import javax.security.sasl.Sasl; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.SaslPropertiesResolver; -import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; -import org.apache.hadoop.util.CombinedIPWhiteList; -import org.apache.hadoop.util.StringUtils; - - -/** - * An implementation of the SaslPropertiesResolver. - * Uses a white list of IPs. - * If the connection's IP address is in the list of IP addresses, the salProperties - * will be unchanged. - * If the connection's IP is not in the list of IP addresses, then QOP for the - * connection will be restricted to "hadoop.rpc.protection.non-whitelist" - * - * Uses 3 IPList implementations together to form an aggregate whitelist. - * 1. ConstantIPList - to check against a set of hardcoded IPs - * 2. Fixed IP List - to check against a list of IP addresses which are specified externally, but - * will not change over runtime. - * 3. Variable IP List - to check against a list of IP addresses which are specified externally and - * could change during runtime. - * A connection IP address will checked against these 3 IP Lists in the order specified above. - * Once a match is found , the IP address is determined to be in whitelist. - * - * The behavior can be configured using a bunch of configuration parameters. - * - */ -public class WhitelistBasedResolver extends SaslPropertiesResolver { - public static final Log LOG = LogFactory.getLog(WhitelistBasedResolver.class); - - private static final String FIXEDWHITELIST_DEFAULT_LOCATION = "/etc/hadoop/fixedwhitelist"; - - private static final String VARIABLEWHITELIST_DEFAULT_LOCATION = "/etc/hadoop/whitelist"; - - /** - * Path to the file to containing subnets and ip addresses to form fixed whitelist. - */ - public static final String HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE = - "hadoop.security.sasl.fixedwhitelist.file"; - /** - * Enables/Disables variable whitelist - */ - public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE = - "hadoop.security.sasl.variablewhitelist.enable"; - /** - * Path to the file to containing subnets and ip addresses to form variable whitelist. - */ - public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE = - "hadoop.security.sasl.variablewhitelist.file"; - /** - * time in seconds by which the variable whitelist file is checked for updates - */ - public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS = - "hadoop.security.sasl.variablewhitelist.cache.secs"; - - /** - * comma separated list containing alternate hadoop.rpc.protection values for - * clients which are not in whitelist - */ - public static final String HADOOP_RPC_PROTECTION_NON_WHITELIST = - "hadoop.rpc.protection.non-whitelist"; - - private CombinedIPWhiteList whiteList; - - private Map saslProps; - - @Override - public void setConf(Configuration conf) { - super.setConf(conf); - String fixedFile = conf.get(HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE, - FIXEDWHITELIST_DEFAULT_LOCATION); - String variableFile = null; - long expiryTime = 0; - - if (conf.getBoolean(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, false)) { - variableFile = conf.get(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE, - VARIABLEWHITELIST_DEFAULT_LOCATION); - expiryTime = - conf.getLong(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS,3600) * 1000; - } - - whiteList = new CombinedIPWhiteList(fixedFile,variableFile,expiryTime); - - this.saslProps = getSaslProperties(conf); - } - - /** - * Identify the Sasl Properties to be used for a connection with a client. - * @param clientAddress client's address - * @return the sasl properties to be used for the connection. - */ - @Override - public Map getServerProperties(InetAddress clientAddress) { - if (clientAddress == null) { - return saslProps; - } - return whiteList.isIn(clientAddress.getHostAddress())?getDefaultProperties():saslProps; - } - - public Map getServerProperties(String clientAddress) throws UnknownHostException { - if (clientAddress == null) { - return saslProps; - } - return getServerProperties(InetAddress.getByName(clientAddress)); - } - - static Map getSaslProperties(Configuration conf) { - Map saslProps =new TreeMap(); - String[] qop = conf.getStrings(HADOOP_RPC_PROTECTION_NON_WHITELIST, - QualityOfProtection.PRIVACY.toString()); - - for (int i=0; i < qop.length; i++) { - qop[i] = QualityOfProtection.valueOf(qop[i].toUpperCase()).getSaslQop(); - } - - saslProps.put(Sasl.QOP, StringUtils.join(",", qop)); - saslProps.put(Sasl.SERVER_AUTH, "true"); - - return saslProps; - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java deleted file mode 100644 index 1343fc6129e..00000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.util; - -/** - * CacheableIPList loads a list of subnets from a file. - * The list is cached and the cache can be refreshed by specifying cache timeout. - * A negative value of cache timeout disables any caching. - * - * Thread safe. - */ - -public class CacheableIPList implements IPList { - private final long cacheTimeout; - private volatile long cacheExpiryTimeStamp; - private volatile FileBasedIPList ipList; - - public CacheableIPList(FileBasedIPList ipList, long cacheTimeout) { - this.cacheTimeout = cacheTimeout; - this.ipList = ipList; - updateCacheExpiryTime(); - } - - /** - * Reloads the ip list - */ - private void reset() { - ipList = ipList.reload(); - updateCacheExpiryTime(); - } - - private void updateCacheExpiryTime() { - if (cacheTimeout < 0) { - cacheExpiryTimeStamp = -1; // no automatic cache expiry. - }else { - cacheExpiryTimeStamp = System.currentTimeMillis() + cacheTimeout; - } - } - - /** - * Refreshes the ip list - */ - public void refresh () { - cacheExpiryTimeStamp = 0; - } - - @Override - public boolean isIn(String ipAddress) { - //is cache expired - //Uses Double Checked Locking using volatile - if (cacheExpiryTimeStamp >= 0 && cacheExpiryTimeStamp < System.currentTimeMillis()) { - synchronized(this) { - //check if cache expired again - if (cacheExpiryTimeStamp < System.currentTimeMillis()) { - reset(); - } - } - } - return ipList.isIn(ipAddress); - } -} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java deleted file mode 100644 index d12c4c11d5d..00000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.util; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class CombinedIPWhiteList implements IPList { - - public static final Log LOG = LogFactory.getLog(CombinedIPWhiteList.class); - private static final String LOCALHOST_IP = "127.0.0.1"; - - private final IPList[] networkLists; - - public CombinedIPWhiteList(String fixedWhiteListFile, - String variableWhiteListFile, long cacheExpiryInSeconds) { - - IPList fixedNetworkList = new FileBasedIPList(fixedWhiteListFile); - if (variableWhiteListFile != null){ - IPList variableNetworkList = new CacheableIPList( - new FileBasedIPList(variableWhiteListFile),cacheExpiryInSeconds); - networkLists = new IPList[] {fixedNetworkList, variableNetworkList}; - } - else { - networkLists = new IPList[] {fixedNetworkList}; - } - } - @Override - public boolean isIn(String ipAddress) { - if (ipAddress == null) { - throw new IllegalArgumentException("ipAddress is null"); - } - - if (LOCALHOST_IP.equals(ipAddress)) { - return true; - } - - for (IPList networkList:networkLists) { - if (networkList.isIn(ipAddress)) { - return true; - } - } - return false; - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java deleted file mode 100644 index 8bfb5d93aef..00000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.util; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * FileBasedIPList loads a list of subnets in CIDR format and ip addresses from a file. - * - * Given an ip address, isIn method returns true if ip belongs to one of the subnets. - * - * Thread safe. - */ - -public class FileBasedIPList implements IPList { - - private static final Log LOG = LogFactory.getLog(FileBasedIPList.class); - - private final String fileName; - private final MachineList addressList; - - public FileBasedIPList(String fileName) { - this.fileName = fileName; - String[] lines = readLines(fileName); - if (lines != null) { - addressList = new MachineList(new HashSet(Arrays.asList(lines))); - } else { - addressList = null; - } - } - - public FileBasedIPList reload() { - return new FileBasedIPList(fileName); - } - - @Override - public boolean isIn(String ipAddress) { - if (ipAddress == null || addressList == null) { - return false; - } - return addressList.includes(ipAddress); - } - - /** - * reads the lines in a file. - * @param fileName - * @return lines in a String array; null if the file does not exist or if the - * file name is null - * @throws IOException - */ - private static String[] readLines(String fileName) { - try { - if (fileName != null) { - File file = new File (fileName); - if (file.exists()) { - FileReader fileReader = new FileReader(file); - BufferedReader bufferedReader = new BufferedReader(fileReader); - List lines = new ArrayList(); - String line = null; - while ((line = bufferedReader.readLine()) != null) { - lines.add(line); - } - bufferedReader.close(); - LOG.debug("Loaded IP list of size = " + lines.size() +" from file = " + fileName); - return(lines.toArray(new String[lines.size()])); - } - else { - LOG.debug("Missing ip list file : "+ fileName); - } - } - } - catch (Throwable t) { - LOG.error(t); - } - return null; - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java deleted file mode 100644 index 3a2616376fb..00000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.util; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -@InterfaceStability.Unstable -@InterfaceAudience.Public -public interface IPList { - - /** - * returns true if the ipAddress is in the IPList. - * @param ipAddress - * @return boolean value indicating whether the ipAddress is in the IPList - */ - public abstract boolean isIn(String ipAddress); -} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java index d1a0870f679..7f2e2c8de7e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java @@ -37,7 +37,7 @@ import com.google.common.net.InetAddresses; /** * Container class which holds a list of ip/host addresses and * answers membership queries. - * + * . * Accepts list of ip addresses, ip addreses in CIDR format and/or * host addresses. */ @@ -71,15 +71,8 @@ public class MachineList { * @param hostEntries comma separated ip/cidr/host addresses */ public MachineList(String hostEntries) { - this(StringUtils.getTrimmedStringCollection(hostEntries)); - } - - /** - * - * @param hostEntries collection of separated ip/cidr/host addresses - */ - public MachineList(Collection hostEntries) { - this(hostEntries, InetAddressFactory.S_INSTANCE); + this(StringUtils.getTrimmedStringCollection(hostEntries), + InetAddressFactory.S_INSTANCE); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java deleted file mode 100644 index 684ef3bf17a..00000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.security; - -import java.io.IOException; -import java.net.InetAddress; -import java.util.Map; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.WhitelistBasedResolver; -import org.apache.hadoop.util.TestFileBasedIPList; - -public class TestWhitelistBasedResolver extends TestCase { - - public static final Map SASL_PRIVACY_PROPS = - WhitelistBasedResolver.getSaslProperties(new Configuration()); - - public void testFixedVariableAndLocalWhiteList() throws IOException { - - String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; - - TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); - - String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; - - TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); - - Configuration conf = new Configuration(); - conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , - "fixedwhitelist.txt"); - - conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, - true); - - conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, - 1); - - conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , - "variablewhitelist.txt"); - - WhitelistBasedResolver wqr = new WhitelistBasedResolver (); - wqr.setConf(conf); - - assertEquals (wqr.getDefaultProperties(), - wqr.getServerProperties(InetAddress.getByName("10.119.103.112"))); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113")); - assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121")); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0")); - assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.222.103.121")); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0")); - assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.113.221.221")); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222")); - assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1")); - - TestFileBasedIPList.removeFile("fixedwhitelist.txt"); - TestFileBasedIPList.removeFile("variablewhitelist.txt"); - } - - - /** - * Add a bunch of subnets and IPSs to the whitelist - * Check for inclusion in whitelist - * Check for exclusion from whitelist - */ - public void testFixedAndLocalWhiteList() throws IOException { - - String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; - - TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); - - String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; - - TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); - - Configuration conf = new Configuration(); - - conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , - "fixedwhitelist.txt"); - - conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, - false); - - conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, - 100); - - conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , - "variablewhitelist.txt"); - - WhitelistBasedResolver wqr = new WhitelistBasedResolver(); - wqr.setConf(conf); - - assertEquals (wqr.getDefaultProperties(), - wqr.getServerProperties(InetAddress.getByName("10.119.103.112"))); - - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113")); - - assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121")); - - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0")); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.222.103.121")); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0")); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.221")); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222")); - assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1"));; - - TestFileBasedIPList.removeFile("fixedwhitelist.txt"); - TestFileBasedIPList.removeFile("variablewhitelist.txt"); - } - - /** - * Add a bunch of subnets and IPSs to the whitelist - * Check for inclusion in whitelist with a null value - */ - public void testNullIPAddress() throws IOException { - - String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; - - TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); - - String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; - - TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); - - Configuration conf = new Configuration(); - conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , - "fixedwhitelist.txt"); - - conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, - true); - - conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, - 100); - - conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , - "variablewhitelist.txt"); - - WhitelistBasedResolver wqr = new WhitelistBasedResolver(); - wqr.setConf(conf); - - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties((InetAddress)null)); - assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties((String)null)); - - TestFileBasedIPList.removeFile("fixedwhitelist.txt"); - TestFileBasedIPList.removeFile("variablewhitelist.txt"); - } -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java deleted file mode 100644 index 3289d786c13..00000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.util; - -import java.io.IOException; - -import org.apache.hadoop.util.CacheableIPList; -import org.apache.hadoop.util.FileBasedIPList; - - -import junit.framework.TestCase; - -public class TestCacheableIPList extends TestCase { - - /** - * Add a bunch of subnets and IPSs to the file - * setup a low cache refresh - * test for inclusion - * Check for exclusion - * Add a bunch of subnets and Ips - * wait for cache timeout. - * test for inclusion - * Check for exclusion - */ - public void testAddWithSleepForCacheTimeout() throws IOException, InterruptedException { - - String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); - - CacheableIPList cipl = new CacheableIPList( - new FileBasedIPList("ips.txt"),100); - - assertFalse("10.113.221.222 is in the list", - cipl.isIn("10.113.221.222")); - assertFalse ("10.222.103.121 is in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - String[]ips2 = {"10.119.103.112", "10.221.102.0/23", - "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); - Thread.sleep(101); - - assertTrue("10.113.221.222 is not in the list", - cipl.isIn("10.113.221.222")); - assertTrue ("10.222.103.121 is not in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - } - - /** - * Add a bunch of subnets and IPSs to the file - * setup a low cache refresh - * test for inclusion - * Check for exclusion - * Remove a bunch of subnets and Ips - * wait for cache timeout. - * test for inclusion - * Check for exclusion - */ - public void testRemovalWithSleepForCacheTimeout() throws IOException, InterruptedException { - - String[] ips = {"10.119.103.112", "10.221.102.0/23", - "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); - - CacheableIPList cipl = new CacheableIPList( - new FileBasedIPList("ips.txt"),100); - - assertTrue("10.113.221.222 is not in the list", - cipl.isIn("10.113.221.222")); - assertTrue ("10.222.103.121 is not in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - String[]ips2 = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); - Thread.sleep(1005); - - assertFalse("10.113.221.222 is in the list", - cipl.isIn("10.113.221.222")); - assertFalse ("10.222.103.121 is in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - } - - /** - * Add a bunch of subnets and IPSs to the file - * setup a low cache refresh - * test for inclusion - * Check for exclusion - * Add a bunch of subnets and Ips - * do a refresh - * test for inclusion - * Check for exclusion - */ - public void testAddWithRefresh() throws IOException, InterruptedException { - - String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); - - CacheableIPList cipl = new CacheableIPList( - new FileBasedIPList("ips.txt"),100); - - assertFalse("10.113.221.222 is in the list", - cipl.isIn("10.113.221.222")); - assertFalse ("10.222.103.121 is in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - String[]ips2 = {"10.119.103.112", "10.221.102.0/23", - "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); - cipl.refresh(); - - assertTrue("10.113.221.222 is not in the list", - cipl.isIn("10.113.221.222")); - assertTrue ("10.222.103.121 is not in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - } - - /** - * Add a bunch of subnets and IPSs to the file - * setup a low cache refresh - * test for inclusion - * Check for exclusion - * Remove a bunch of subnets and Ips - * wait for cache timeout. - * test for inclusion - * Check for exclusion - */ - public void testRemovalWithRefresh() throws IOException, InterruptedException { - - String[] ips = {"10.119.103.112", "10.221.102.0/23", - "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); - - CacheableIPList cipl = new CacheableIPList( - new FileBasedIPList("ips.txt"),100); - - assertTrue("10.113.221.222 is not in the list", - cipl.isIn("10.113.221.222")); - assertTrue ("10.222.103.121 is not in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - String[]ips2 = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; - - TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); - cipl.refresh(); - - assertFalse("10.113.221.222 is in the list", - cipl.isIn("10.113.221.222")); - assertFalse ("10.222.103.121 is in the list", - cipl.isIn("10.222.103.121")); - - TestFileBasedIPList.removeFile("ips.txt"); - } - - - -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java deleted file mode 100644 index 0e79fd1bbab..00000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.util; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.util.FileBasedIPList; -import org.apache.hadoop.util.IPList; -import org.junit.After; -import org.junit.Test; - -import junit.framework.TestCase; - -public class TestFileBasedIPList extends TestCase { - - @After - public void tearDown() { - removeFile("ips.txt"); - } - - /** - * Add a bunch of IPS to the file - * Check for inclusion - * Check for exclusion - */ - @Test - public void testSubnetsAndIPs() throws IOException { - - String[] ips = {"10.119.103.112", "10.221.102.0/23"}; - - createFileWithEntries ("ips.txt", ips); - - IPList ipList = new FileBasedIPList("ips.txt"); - - assertTrue ("10.119.103.112 is not in the list", - ipList.isIn("10.119.103.112")); - assertFalse ("10.119.103.113 is in the list", - ipList.isIn("10.119.103.113")); - - assertTrue ("10.221.102.0 is not in the list", - ipList.isIn("10.221.102.0")); - assertTrue ("10.221.102.1 is not in the list", - ipList.isIn("10.221.102.1")); - assertTrue ("10.221.103.1 is not in the list", - ipList.isIn("10.221.103.1")); - assertTrue ("10.221.103.255 is not in the list", - ipList.isIn("10.221.103.255")); - assertFalse("10.221.104.0 is in the list", - ipList.isIn("10.221.104.0")); - assertFalse("10.221.104.1 is in the list", - ipList.isIn("10.221.104.1")); - } - - /** - * Add a bunch of IPS to the file - * Check for inclusion - * Check for exclusion - */ - @Test - public void testNullIP() throws IOException { - - String[] ips = {"10.119.103.112", "10.221.102.0/23"}; - createFileWithEntries ("ips.txt", ips); - - IPList ipList = new FileBasedIPList("ips.txt"); - - assertFalse ("Null Ip is in the list", - ipList.isIn(null)); - } - - /** - * Add a bunch of subnets and IPSs to the file - * Check for inclusion - * Check for exclusion - */ - @Test - public void testWithMultipleSubnetAndIPs() throws IOException { - - String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.222.0.0/16", - "10.113.221.221"}; - - createFileWithEntries ("ips.txt", ips); - - IPList ipList = new FileBasedIPList("ips.txt"); - - assertTrue ("10.119.103.112 is not in the list", - ipList.isIn("10.119.103.112")); - assertFalse ("10.119.103.113 is in the list", - ipList.isIn("10.119.103.113")); - - assertTrue ("10.221.103.121 is not in the list", - ipList.isIn("10.221.103.121")); - assertFalse("10.221.104.0 is in the list", - ipList.isIn("10.221.104.0")); - - assertTrue ("10.222.103.121 is not in the list", - ipList.isIn("10.222.103.121")); - assertFalse("10.223.104.0 is in the list", - ipList.isIn("10.223.104.0")); - - assertTrue ("10.113.221.221 is not in the list", - ipList.isIn("10.113.221.221")); - assertFalse("10.113.221.222 is in the list", - ipList.isIn("10.113.221.222")); - } - - /** - * Do not specify the file - * test for inclusion - * should be true as if the feature is turned off - */ - public void testFileNotSpecified() { - - IPList ipl = new FileBasedIPList(null); - - assertFalse("110.113.221.222 is in the list", - ipl.isIn("110.113.221.222")); - } - - /** - * Specify a non existent file - * test for inclusion - * should be true as if the feature is turned off - */ - public void testFileMissing() { - - IPList ipl = new FileBasedIPList("missingips.txt"); - - assertFalse("110.113.221.222 is in the list", - ipl.isIn("110.113.221.222")); - } - - /** - * Specify an existing file, but empty - * test for inclusion - * should be true as if the feature is turned off - */ - public void testWithEmptyList() throws IOException { - String[] ips = {}; - - createFileWithEntries ("ips.txt", ips); - IPList ipl = new FileBasedIPList("ips.txt"); - - assertFalse("110.113.221.222 is in the list", - ipl.isIn("110.113.221.222")); - } - - /** - * Specify an existing file, but ips in wrong format - * test for inclusion - * should be true as if the feature is turned off - */ - public void testForBadFIle() throws IOException { - String[] ips = { "10.221.102/23"}; - - createFileWithEntries ("ips.txt", ips); - - try { - new FileBasedIPList("ips.txt"); - fail(); - } catch (Exception e) { - //expects Exception - } - } - - /** - * Add a bunch of subnets and IPSs to the file. Keep one entry wrong. - * The good entries will still be used. - * Check for inclusion with good entries - * Check for exclusion - */ - public void testWithAWrongEntry() throws IOException { - - String[] ips = {"10.119.103.112", "10.221.102/23", "10.221.204.1/23"}; - - createFileWithEntries ("ips.txt", ips); - - try { - new FileBasedIPList("ips.txt"); - fail(); - } catch (Exception e) { - //expects Exception - } - } - - public static void createFileWithEntries(String fileName, String[] ips) - throws IOException { - FileUtils.writeLines(new File(fileName), Arrays.asList(ips)); - } - - public static void removeFile(String fileName) { - File file = new File(fileName); - if (file.exists()) { - new File(fileName).delete(); - } - } -} From 86840834ed822527196a7eca117648be4df43292 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 17 Aug 2014 19:02:45 +0000 Subject: [PATCH 57/67] HADOOP-10335. An ip whilelist based implementation to resolve Sasl properties per connection. (Contributed by Benoy Antony) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618503 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../security/WhitelistBasedResolver.java | 149 ++++++++++++ .../apache/hadoop/util/CacheableIPList.java | 76 +++++++ .../hadoop/util/CombinedIPWhiteList.java | 60 +++++ .../apache/hadoop/util/FileBasedIPList.java | 102 +++++++++ .../java/org/apache/hadoop/util/IPList.java | 33 +++ .../org/apache/hadoop/util/MachineList.java | 13 +- .../security/TestWhitelistBasedResolver.java | 163 +++++++++++++ .../hadoop/util/TestCacheableIPList.java | 188 +++++++++++++++ .../hadoop/util/TestFileBasedIPList.java | 215 ++++++++++++++++++ 10 files changed, 999 insertions(+), 3 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d0168c2d5bc..fb0578f7c50 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -524,6 +524,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10650. Add ability to specify a reverse ACL (black list) of users and groups. (Benoy Antony via Arpit Agarwal) + HADOOP-10335. An ip whilelist based implementation to resolve Sasl + properties per connection. (Benoy Antony via Arpit Agarwal) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java new file mode 100644 index 00000000000..dc0815ed768 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; +import java.util.TreeMap; + +import javax.security.sasl.Sasl; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SaslPropertiesResolver; +import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; +import org.apache.hadoop.util.CombinedIPWhiteList; +import org.apache.hadoop.util.StringUtils; + + +/** + * An implementation of the SaslPropertiesResolver. + * Uses a white list of IPs. + * If the connection's IP address is in the list of IP addresses, the salProperties + * will be unchanged. + * If the connection's IP is not in the list of IP addresses, then QOP for the + * connection will be restricted to "hadoop.rpc.protection.non-whitelist" + * + * Uses 3 IPList implementations together to form an aggregate whitelist. + * 1. ConstantIPList - to check against a set of hardcoded IPs + * 2. Fixed IP List - to check against a list of IP addresses which are specified externally, but + * will not change over runtime. + * 3. Variable IP List - to check against a list of IP addresses which are specified externally and + * could change during runtime. + * A connection IP address will checked against these 3 IP Lists in the order specified above. + * Once a match is found , the IP address is determined to be in whitelist. + * + * The behavior can be configured using a bunch of configuration parameters. + * + */ +public class WhitelistBasedResolver extends SaslPropertiesResolver { + public static final Log LOG = LogFactory.getLog(WhitelistBasedResolver.class); + + private static final String FIXEDWHITELIST_DEFAULT_LOCATION = "/etc/hadoop/fixedwhitelist"; + + private static final String VARIABLEWHITELIST_DEFAULT_LOCATION = "/etc/hadoop/whitelist"; + + /** + * Path to the file to containing subnets and ip addresses to form fixed whitelist. + */ + public static final String HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE = + "hadoop.security.sasl.fixedwhitelist.file"; + /** + * Enables/Disables variable whitelist + */ + public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE = + "hadoop.security.sasl.variablewhitelist.enable"; + /** + * Path to the file to containing subnets and ip addresses to form variable whitelist. + */ + public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE = + "hadoop.security.sasl.variablewhitelist.file"; + /** + * time in seconds by which the variable whitelist file is checked for updates + */ + public static final String HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS = + "hadoop.security.sasl.variablewhitelist.cache.secs"; + + /** + * comma separated list containing alternate hadoop.rpc.protection values for + * clients which are not in whitelist + */ + public static final String HADOOP_RPC_PROTECTION_NON_WHITELIST = + "hadoop.rpc.protection.non-whitelist"; + + private CombinedIPWhiteList whiteList; + + private Map saslProps; + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + String fixedFile = conf.get(HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE, + FIXEDWHITELIST_DEFAULT_LOCATION); + String variableFile = null; + long expiryTime = 0; + + if (conf.getBoolean(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, false)) { + variableFile = conf.get(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE, + VARIABLEWHITELIST_DEFAULT_LOCATION); + expiryTime = + conf.getLong(HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS,3600) * 1000; + } + + whiteList = new CombinedIPWhiteList(fixedFile,variableFile,expiryTime); + + this.saslProps = getSaslProperties(conf); + } + + /** + * Identify the Sasl Properties to be used for a connection with a client. + * @param clientAddress client's address + * @return the sasl properties to be used for the connection. + */ + @Override + public Map getServerProperties(InetAddress clientAddress) { + if (clientAddress == null) { + return saslProps; + } + return whiteList.isIn(clientAddress.getHostAddress())?getDefaultProperties():saslProps; + } + + public Map getServerProperties(String clientAddress) throws UnknownHostException { + if (clientAddress == null) { + return saslProps; + } + return getServerProperties(InetAddress.getByName(clientAddress)); + } + + static Map getSaslProperties(Configuration conf) { + Map saslProps =new TreeMap(); + String[] qop = conf.getStrings(HADOOP_RPC_PROTECTION_NON_WHITELIST, + QualityOfProtection.PRIVACY.toString()); + + for (int i=0; i < qop.length; i++) { + qop[i] = QualityOfProtection.valueOf(qop[i].toUpperCase()).getSaslQop(); + } + + saslProps.put(Sasl.QOP, StringUtils.join(",", qop)); + saslProps.put(Sasl.SERVER_AUTH, "true"); + + return saslProps; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java new file mode 100644 index 00000000000..1343fc6129e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CacheableIPList.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +/** + * CacheableIPList loads a list of subnets from a file. + * The list is cached and the cache can be refreshed by specifying cache timeout. + * A negative value of cache timeout disables any caching. + * + * Thread safe. + */ + +public class CacheableIPList implements IPList { + private final long cacheTimeout; + private volatile long cacheExpiryTimeStamp; + private volatile FileBasedIPList ipList; + + public CacheableIPList(FileBasedIPList ipList, long cacheTimeout) { + this.cacheTimeout = cacheTimeout; + this.ipList = ipList; + updateCacheExpiryTime(); + } + + /** + * Reloads the ip list + */ + private void reset() { + ipList = ipList.reload(); + updateCacheExpiryTime(); + } + + private void updateCacheExpiryTime() { + if (cacheTimeout < 0) { + cacheExpiryTimeStamp = -1; // no automatic cache expiry. + }else { + cacheExpiryTimeStamp = System.currentTimeMillis() + cacheTimeout; + } + } + + /** + * Refreshes the ip list + */ + public void refresh () { + cacheExpiryTimeStamp = 0; + } + + @Override + public boolean isIn(String ipAddress) { + //is cache expired + //Uses Double Checked Locking using volatile + if (cacheExpiryTimeStamp >= 0 && cacheExpiryTimeStamp < System.currentTimeMillis()) { + synchronized(this) { + //check if cache expired again + if (cacheExpiryTimeStamp < System.currentTimeMillis()) { + reset(); + } + } + } + return ipList.isIn(ipAddress); + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java new file mode 100644 index 00000000000..d12c4c11d5d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPWhiteList.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class CombinedIPWhiteList implements IPList { + + public static final Log LOG = LogFactory.getLog(CombinedIPWhiteList.class); + private static final String LOCALHOST_IP = "127.0.0.1"; + + private final IPList[] networkLists; + + public CombinedIPWhiteList(String fixedWhiteListFile, + String variableWhiteListFile, long cacheExpiryInSeconds) { + + IPList fixedNetworkList = new FileBasedIPList(fixedWhiteListFile); + if (variableWhiteListFile != null){ + IPList variableNetworkList = new CacheableIPList( + new FileBasedIPList(variableWhiteListFile),cacheExpiryInSeconds); + networkLists = new IPList[] {fixedNetworkList, variableNetworkList}; + } + else { + networkLists = new IPList[] {fixedNetworkList}; + } + } + @Override + public boolean isIn(String ipAddress) { + if (ipAddress == null) { + throw new IllegalArgumentException("ipAddress is null"); + } + + if (LOCALHOST_IP.equals(ipAddress)) { + return true; + } + + for (IPList networkList:networkLists) { + if (networkList.isIn(ipAddress)) { + return true; + } + } + return false; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java new file mode 100644 index 00000000000..8bfb5d93aef --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * FileBasedIPList loads a list of subnets in CIDR format and ip addresses from a file. + * + * Given an ip address, isIn method returns true if ip belongs to one of the subnets. + * + * Thread safe. + */ + +public class FileBasedIPList implements IPList { + + private static final Log LOG = LogFactory.getLog(FileBasedIPList.class); + + private final String fileName; + private final MachineList addressList; + + public FileBasedIPList(String fileName) { + this.fileName = fileName; + String[] lines = readLines(fileName); + if (lines != null) { + addressList = new MachineList(new HashSet(Arrays.asList(lines))); + } else { + addressList = null; + } + } + + public FileBasedIPList reload() { + return new FileBasedIPList(fileName); + } + + @Override + public boolean isIn(String ipAddress) { + if (ipAddress == null || addressList == null) { + return false; + } + return addressList.includes(ipAddress); + } + + /** + * reads the lines in a file. + * @param fileName + * @return lines in a String array; null if the file does not exist or if the + * file name is null + * @throws IOException + */ + private static String[] readLines(String fileName) { + try { + if (fileName != null) { + File file = new File (fileName); + if (file.exists()) { + FileReader fileReader = new FileReader(file); + BufferedReader bufferedReader = new BufferedReader(fileReader); + List lines = new ArrayList(); + String line = null; + while ((line = bufferedReader.readLine()) != null) { + lines.add(line); + } + bufferedReader.close(); + LOG.debug("Loaded IP list of size = " + lines.size() +" from file = " + fileName); + return(lines.toArray(new String[lines.size()])); + } + else { + LOG.debug("Missing ip list file : "+ fileName); + } + } + } + catch (Throwable t) { + LOG.error(t); + } + return null; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java new file mode 100644 index 00000000000..3a2616376fb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceStability.Unstable +@InterfaceAudience.Public +public interface IPList { + + /** + * returns true if the ipAddress is in the IPList. + * @param ipAddress + * @return boolean value indicating whether the ipAddress is in the IPList + */ + public abstract boolean isIn(String ipAddress); +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java index 7f2e2c8de7e..d1a0870f679 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java @@ -37,7 +37,7 @@ import com.google.common.net.InetAddresses; /** * Container class which holds a list of ip/host addresses and * answers membership queries. - * . + * * Accepts list of ip addresses, ip addreses in CIDR format and/or * host addresses. */ @@ -71,8 +71,15 @@ public class MachineList { * @param hostEntries comma separated ip/cidr/host addresses */ public MachineList(String hostEntries) { - this(StringUtils.getTrimmedStringCollection(hostEntries), - InetAddressFactory.S_INSTANCE); + this(StringUtils.getTrimmedStringCollection(hostEntries)); + } + + /** + * + * @param hostEntries collection of separated ip/cidr/host addresses + */ + public MachineList(Collection hostEntries) { + this(hostEntries, InetAddressFactory.S_INSTANCE); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java new file mode 100644 index 00000000000..684ef3bf17a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.WhitelistBasedResolver; +import org.apache.hadoop.util.TestFileBasedIPList; + +public class TestWhitelistBasedResolver extends TestCase { + + public static final Map SASL_PRIVACY_PROPS = + WhitelistBasedResolver.getSaslProperties(new Configuration()); + + public void testFixedVariableAndLocalWhiteList() throws IOException { + + String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; + + TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); + + String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); + + Configuration conf = new Configuration(); + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , + "fixedwhitelist.txt"); + + conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, + true); + + conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, + 1); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , + "variablewhitelist.txt"); + + WhitelistBasedResolver wqr = new WhitelistBasedResolver (); + wqr.setConf(conf); + + assertEquals (wqr.getDefaultProperties(), + wqr.getServerProperties(InetAddress.getByName("10.119.103.112"))); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.222.103.121")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.113.221.221")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1")); + + TestFileBasedIPList.removeFile("fixedwhitelist.txt"); + TestFileBasedIPList.removeFile("variablewhitelist.txt"); + } + + + /** + * Add a bunch of subnets and IPSs to the whitelist + * Check for inclusion in whitelist + * Check for exclusion from whitelist + */ + public void testFixedAndLocalWhiteList() throws IOException { + + String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; + + TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); + + String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); + + Configuration conf = new Configuration(); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , + "fixedwhitelist.txt"); + + conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, + false); + + conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, + 100); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , + "variablewhitelist.txt"); + + WhitelistBasedResolver wqr = new WhitelistBasedResolver(); + wqr.setConf(conf); + + assertEquals (wqr.getDefaultProperties(), + wqr.getServerProperties(InetAddress.getByName("10.119.103.112"))); + + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113")); + + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121")); + + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.222.103.121")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.221")); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222")); + assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1"));; + + TestFileBasedIPList.removeFile("fixedwhitelist.txt"); + TestFileBasedIPList.removeFile("variablewhitelist.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the whitelist + * Check for inclusion in whitelist with a null value + */ + public void testNullIPAddress() throws IOException { + + String[] fixedIps = {"10.119.103.112", "10.221.102.0/23"}; + + TestFileBasedIPList.createFileWithEntries ("fixedwhitelist.txt", fixedIps); + + String[] variableIps = {"10.222.0.0/16", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("variablewhitelist.txt", variableIps); + + Configuration conf = new Configuration(); + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_FIXEDWHITELIST_FILE , + "fixedwhitelist.txt"); + + conf.setBoolean(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_ENABLE, + true); + + conf.setLong(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_CACHE_SECS, + 100); + + conf.set(WhitelistBasedResolver.HADOOP_SECURITY_SASL_VARIABLEWHITELIST_FILE , + "variablewhitelist.txt"); + + WhitelistBasedResolver wqr = new WhitelistBasedResolver(); + wqr.setConf(conf); + + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties((InetAddress)null)); + assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties((String)null)); + + TestFileBasedIPList.removeFile("fixedwhitelist.txt"); + TestFileBasedIPList.removeFile("variablewhitelist.txt"); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java new file mode 100644 index 00000000000..3289d786c13 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCacheableIPList.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.io.IOException; + +import org.apache.hadoop.util.CacheableIPList; +import org.apache.hadoop.util.FileBasedIPList; + + +import junit.framework.TestCase; + +public class TestCacheableIPList extends TestCase { + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Add a bunch of subnets and Ips + * wait for cache timeout. + * test for inclusion + * Check for exclusion + */ + public void testAddWithSleepForCacheTimeout() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + Thread.sleep(101); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Remove a bunch of subnets and Ips + * wait for cache timeout. + * test for inclusion + * Check for exclusion + */ + public void testRemovalWithSleepForCacheTimeout() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + Thread.sleep(1005); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Add a bunch of subnets and Ips + * do a refresh + * test for inclusion + * Check for exclusion + */ + public void testAddWithRefresh() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + cipl.refresh(); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + /** + * Add a bunch of subnets and IPSs to the file + * setup a low cache refresh + * test for inclusion + * Check for exclusion + * Remove a bunch of subnets and Ips + * wait for cache timeout. + * test for inclusion + * Check for exclusion + */ + public void testRemovalWithRefresh() throws IOException, InterruptedException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", + "10.222.0.0/16", "10.113.221.221", "10.113.221.222"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips); + + CacheableIPList cipl = new CacheableIPList( + new FileBasedIPList("ips.txt"),100); + + assertTrue("10.113.221.222 is not in the list", + cipl.isIn("10.113.221.222")); + assertTrue ("10.222.103.121 is not in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + String[]ips2 = {"10.119.103.112", "10.221.102.0/23", "10.113.221.221"}; + + TestFileBasedIPList.createFileWithEntries ("ips.txt", ips2); + cipl.refresh(); + + assertFalse("10.113.221.222 is in the list", + cipl.isIn("10.113.221.222")); + assertFalse ("10.222.103.121 is in the list", + cipl.isIn("10.222.103.121")); + + TestFileBasedIPList.removeFile("ips.txt"); + } + + + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java new file mode 100644 index 00000000000..0e79fd1bbab --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFileBasedIPList.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.util.FileBasedIPList; +import org.apache.hadoop.util.IPList; +import org.junit.After; +import org.junit.Test; + +import junit.framework.TestCase; + +public class TestFileBasedIPList extends TestCase { + + @After + public void tearDown() { + removeFile("ips.txt"); + } + + /** + * Add a bunch of IPS to the file + * Check for inclusion + * Check for exclusion + */ + @Test + public void testSubnetsAndIPs() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23"}; + + createFileWithEntries ("ips.txt", ips); + + IPList ipList = new FileBasedIPList("ips.txt"); + + assertTrue ("10.119.103.112 is not in the list", + ipList.isIn("10.119.103.112")); + assertFalse ("10.119.103.113 is in the list", + ipList.isIn("10.119.103.113")); + + assertTrue ("10.221.102.0 is not in the list", + ipList.isIn("10.221.102.0")); + assertTrue ("10.221.102.1 is not in the list", + ipList.isIn("10.221.102.1")); + assertTrue ("10.221.103.1 is not in the list", + ipList.isIn("10.221.103.1")); + assertTrue ("10.221.103.255 is not in the list", + ipList.isIn("10.221.103.255")); + assertFalse("10.221.104.0 is in the list", + ipList.isIn("10.221.104.0")); + assertFalse("10.221.104.1 is in the list", + ipList.isIn("10.221.104.1")); + } + + /** + * Add a bunch of IPS to the file + * Check for inclusion + * Check for exclusion + */ + @Test + public void testNullIP() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23"}; + createFileWithEntries ("ips.txt", ips); + + IPList ipList = new FileBasedIPList("ips.txt"); + + assertFalse ("Null Ip is in the list", + ipList.isIn(null)); + } + + /** + * Add a bunch of subnets and IPSs to the file + * Check for inclusion + * Check for exclusion + */ + @Test + public void testWithMultipleSubnetAndIPs() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102.0/23", "10.222.0.0/16", + "10.113.221.221"}; + + createFileWithEntries ("ips.txt", ips); + + IPList ipList = new FileBasedIPList("ips.txt"); + + assertTrue ("10.119.103.112 is not in the list", + ipList.isIn("10.119.103.112")); + assertFalse ("10.119.103.113 is in the list", + ipList.isIn("10.119.103.113")); + + assertTrue ("10.221.103.121 is not in the list", + ipList.isIn("10.221.103.121")); + assertFalse("10.221.104.0 is in the list", + ipList.isIn("10.221.104.0")); + + assertTrue ("10.222.103.121 is not in the list", + ipList.isIn("10.222.103.121")); + assertFalse("10.223.104.0 is in the list", + ipList.isIn("10.223.104.0")); + + assertTrue ("10.113.221.221 is not in the list", + ipList.isIn("10.113.221.221")); + assertFalse("10.113.221.222 is in the list", + ipList.isIn("10.113.221.222")); + } + + /** + * Do not specify the file + * test for inclusion + * should be true as if the feature is turned off + */ + public void testFileNotSpecified() { + + IPList ipl = new FileBasedIPList(null); + + assertFalse("110.113.221.222 is in the list", + ipl.isIn("110.113.221.222")); + } + + /** + * Specify a non existent file + * test for inclusion + * should be true as if the feature is turned off + */ + public void testFileMissing() { + + IPList ipl = new FileBasedIPList("missingips.txt"); + + assertFalse("110.113.221.222 is in the list", + ipl.isIn("110.113.221.222")); + } + + /** + * Specify an existing file, but empty + * test for inclusion + * should be true as if the feature is turned off + */ + public void testWithEmptyList() throws IOException { + String[] ips = {}; + + createFileWithEntries ("ips.txt", ips); + IPList ipl = new FileBasedIPList("ips.txt"); + + assertFalse("110.113.221.222 is in the list", + ipl.isIn("110.113.221.222")); + } + + /** + * Specify an existing file, but ips in wrong format + * test for inclusion + * should be true as if the feature is turned off + */ + public void testForBadFIle() throws IOException { + String[] ips = { "10.221.102/23"}; + + createFileWithEntries ("ips.txt", ips); + + try { + new FileBasedIPList("ips.txt"); + fail(); + } catch (Exception e) { + //expects Exception + } + } + + /** + * Add a bunch of subnets and IPSs to the file. Keep one entry wrong. + * The good entries will still be used. + * Check for inclusion with good entries + * Check for exclusion + */ + public void testWithAWrongEntry() throws IOException { + + String[] ips = {"10.119.103.112", "10.221.102/23", "10.221.204.1/23"}; + + createFileWithEntries ("ips.txt", ips); + + try { + new FileBasedIPList("ips.txt"); + fail(); + } catch (Exception e) { + //expects Exception + } + } + + public static void createFileWithEntries(String fileName, String[] ips) + throws IOException { + FileUtils.writeLines(new File(fileName), Arrays.asList(ips)); + } + + public static void removeFile(String fileName) { + File file = new File(fileName); + if (file.exists()) { + new File(fileName).delete(); + } + } +} From 519c4be95a091a072302e9ae52c2d221d80037a8 Mon Sep 17 00:00:00 2001 From: Jian He Date: Mon, 18 Aug 2014 06:08:45 +0000 Subject: [PATCH 58/67] YARN-2411. Support simple user and group mappings to queues. Contributed by Ram Venkatesh git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618542 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../conf/capacity-scheduler.xml | 23 ++ .../scheduler/capacity/CapacityScheduler.java | 121 ++++++++- .../CapacitySchedulerConfiguration.java | 119 ++++++++- .../scheduler/capacity/TestQueueMappings.java | 240 ++++++++++++++++++ 5 files changed, 492 insertions(+), 14 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index aa217edaec0..0bb4a8ccd68 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -47,6 +47,9 @@ Release 2.6.0 - UNRELEASED YARN-2378. Added support for moving applications across queues in CapacityScheduler. (Subramaniam Venkatraman Krishnan via jianhe) + YARN-2411. Support simple user and group mappings to queues. (Ram Venkatesh + via jianhe) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml index 2bed464bcc6..30f4eb96c6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml @@ -108,4 +108,27 @@ + + yarn.scheduler.capacity.queue-mappings + + + A list of mappings that will be used to assign jobs to queues + The syntax for this list is [u|g]:[name]:[queue_name][,next mapping]* + Typically this list will be used to map users to queues, + for example, u:%user:%user maps all users to queues with the same name + as the user. + + + + + yarn.scheduler.capacity.queue-mappings-override.enable + false + + If a queue mapping is present, will it override the value specified + by the user? This can be used by administrators to place jobs in queues + that are different than the one specified by the user. + The default is false. + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 1b31a1a6bbb..e979c5c2320 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import com.google.common.base.Preconditions; - import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -41,6 +39,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -59,10 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.*; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -77,6 +73,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResour import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping.MappingType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -94,6 +92,7 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; @LimitedPrivate("yarn") @Evolving @@ -199,6 +198,16 @@ public class CapacityScheduler extends + ".scheduling-interval-ms"; private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; + private boolean overrideWithQueueMappings = false; + private List mappings = new ArrayList(); + private Groups groups; + + @VisibleForTesting + public synchronized String getMappedQueueForTest(String user) + throws IOException { + return getMappedQueue(user); + } + public CapacityScheduler() { super(CapacityScheduler.class.getName()); } @@ -263,7 +272,6 @@ public class CapacityScheduler extends this.applications = new ConcurrentHashMap>(); - initializeQueues(this.conf); scheduleAsynchronously = this.conf.getScheduleAynschronously(); @@ -402,7 +410,32 @@ public class CapacityScheduler extends } } private static final QueueHook noop = new QueueHook(); - + + private void initializeQueueMappings() throws IOException { + overrideWithQueueMappings = conf.getOverrideWithQueueMappings(); + LOG.info("Initialized queue mappings, override: " + + overrideWithQueueMappings); + // Get new user/group mappings + List newMappings = conf.getQueueMappings(); + //check if mappings refer to valid queues + for (QueueMapping mapping : newMappings) { + if (!mapping.queue.equals(CURRENT_USER_MAPPING) && + !mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { + CSQueue queue = queues.get(mapping.queue); + if (queue == null || !(queue instanceof LeafQueue)) { + throw new IOException( + "mapping contains invalid or non-leaf queue " + mapping.queue); + } + } + } + //apply the new mappings since they are valid + mappings = newMappings; + // initialize groups if mappings are present + if (mappings.size() > 0) { + groups = new Groups(conf); + } + } + @Lock(CapacityScheduler.class) private void initializeQueues(CapacitySchedulerConfiguration conf) throws IOException { @@ -410,7 +443,9 @@ public class CapacityScheduler extends root = parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, noop); + LOG.info("Initialized root queue " + root); + initializeQueueMappings(); } @Lock(CapacityScheduler.class) @@ -430,6 +465,7 @@ public class CapacityScheduler extends // Re-configure queues root.reinitialize(newRoot, clusterResource); + initializeQueueMappings(); } /** @@ -517,12 +553,73 @@ public class CapacityScheduler extends } synchronized CSQueue getQueue(String queueName) { + if (queueName == null) { + return null; + } return queues.get(queueName); } + private static final String CURRENT_USER_MAPPING = "%user"; + + private static final String PRIMARY_GROUP_MAPPING = "%primary_group"; + + private String getMappedQueue(String user) throws IOException { + for (QueueMapping mapping : mappings) { + if (mapping.type == MappingType.USER) { + if (mapping.source.equals(CURRENT_USER_MAPPING)) { + if (mapping.queue.equals(CURRENT_USER_MAPPING)) { + return user; + } + else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { + return groups.getGroups(user).get(0); + } + else { + return mapping.queue; + } + } + if (user.equals(mapping.source)) { + return mapping.queue; + } + } + if (mapping.type == MappingType.GROUP) { + for (String userGroups : groups.getGroups(user)) { + if (userGroups.equals(mapping.source)) { + return mapping.queue; + } + } + } + } + return null; + } + private synchronized void addApplication(ApplicationId applicationId, - String queueName, String user, boolean isAppRecovering) { - // santiy checks. + String queueName, String user, boolean isAppRecovering) { + + if (mappings != null && mappings.size() > 0) { + try { + String mappedQueue = getMappedQueue(user); + if (mappedQueue != null) { + // We have a mapping, should we use it? + if (queueName.equals(YarnConfiguration.DEFAULT_QUEUE_NAME) + || overrideWithQueueMappings) { + LOG.info("Application " + applicationId + " user " + user + + " mapping [" + queueName + "] to [" + mappedQueue + + "] override " + overrideWithQueueMappings); + queueName = mappedQueue; + RMApp rmApp = rmContext.getRMApps().get(applicationId); + rmApp.setQueue(queueName); + } + } + } catch (IOException ioex) { + String message = "Failed to submit application " + applicationId + + " submitted by user " + user + " reason: " + ioex.getMessage(); + this.rmContext.getDispatcher().getEventHandler() + .handle(new RMAppRejectedEvent(applicationId, message)); + return; + } + } + + // sanity checks. CSQueue queue = getQueue(queueName); if (queue == null) { String message = "Application " + applicationId + @@ -902,8 +999,8 @@ public class CapacityScheduler extends { AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; addApplication(appAddedEvent.getApplicationId(), - appAddedEvent.getQueue(), appAddedEvent.getUser(), - appAddedEvent.getIsAppRecovering()); + appAddedEvent.getQueue(), + appAddedEvent.getUser(), appAddedEvent.getIsAppRecovering()); } break; case APP_REMOVED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 6fe695ecda2..af6bdc301ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -18,8 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -145,6 +144,44 @@ public class CapacitySchedulerConfiguration extends Configuration { @Private public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false; + + @Private + public static final String QUEUE_MAPPING = PREFIX + "queue-mappings"; + + @Private + public static final String ENABLE_QUEUE_MAPPING_OVERRIDE = QUEUE_MAPPING + "-override.enable"; + + @Private + public static final boolean DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE = false; + + @Private + public static class QueueMapping { + + public enum MappingType { + + USER("u"), + GROUP("g"); + private final String type; + private MappingType(String type) { + this.type = type; + } + + public String toString() { + return type; + } + + }; + + MappingType type; + String source; + String queue; + + public QueueMapping(MappingType type, String source, String queue) { + this.type = type; + this.source = source; + this.queue = queue; + } + } public CapacitySchedulerConfiguration() { this(new Configuration()); @@ -378,4 +415,82 @@ public class CapacitySchedulerConfiguration extends Configuration { setBoolean(SCHEDULE_ASYNCHRONOUSLY_ENABLE, async); } + public boolean getOverrideWithQueueMappings() { + return getBoolean(ENABLE_QUEUE_MAPPING_OVERRIDE, + DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE); + } + + /** + * Returns a collection of strings, trimming leading and trailing whitespeace + * on each value + * + * @param str + * String to parse + * @param delim + * delimiter to separate the values + * @return Collection of parsed elements. + */ + private static Collection getTrimmedStringCollection(String str, + String delim) { + List values = new ArrayList(); + if (str == null) + return values; + StringTokenizer tokenizer = new StringTokenizer(str, delim); + while (tokenizer.hasMoreTokens()) { + String next = tokenizer.nextToken(); + if (next == null || next.trim().isEmpty()) { + continue; + } + values.add(next.trim()); + } + return values; + } + + /** + * Get user/group mappings to queues. + * + * @return user/groups mappings or null on illegal configs + */ + public List getQueueMappings() { + List mappings = + new ArrayList(); + Collection mappingsString = + getTrimmedStringCollection(QUEUE_MAPPING); + for (String mappingValue : mappingsString) { + String[] mapping = + getTrimmedStringCollection(mappingValue, ":") + .toArray(new String[] {}); + if (mapping.length != 3 || mapping[1].length() == 0 + || mapping[2].length() == 0) { + throw new IllegalArgumentException( + "Illegal queue mapping " + mappingValue); + } + + QueueMapping m; + try { + QueueMapping.MappingType mappingType; + if (mapping[0].equals("u")) { + mappingType = QueueMapping.MappingType.USER; + } else if (mapping[0].equals("g")) { + mappingType = QueueMapping.MappingType.GROUP; + } else { + throw new IllegalArgumentException( + "unknown mapping prefix " + mapping[0]); + } + m = new QueueMapping( + mappingType, + mapping[1], + mapping[2]); + } catch (Throwable t) { + throw new IllegalArgumentException( + "Illegal queue mapping " + mappingValue); + } + + if (m != null) { + mappings.add(m); + } + } + + return mappings; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java new file mode 100644 index 00000000000..f573f43f067 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.io.IOException; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class TestQueueMappings { + + private static final Log LOG = LogFactory.getLog(TestQueueMappings.class); + + private static final String Q1 = "q1"; + private static final String Q2 = "q2"; + + private final static String Q1_PATH = + CapacitySchedulerConfiguration.ROOT + "." + Q1; + private final static String Q2_PATH = + CapacitySchedulerConfiguration.ROOT + "." + Q2; + + private MockRM resourceManager; + + @After + public void tearDown() throws Exception { + if (resourceManager != null) { + LOG.info("Stopping the resource manager"); + resourceManager.stop(); + } + } + + private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { Q1, Q2 }); + + conf.setCapacity(Q1_PATH, 10); + conf.setCapacity(Q2_PATH, 90); + + LOG.info("Setup top-level queues q1 and q2"); + } + + @Test (timeout = 60000) + public void testQueueMapping() throws Exception { + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + YarnConfiguration conf = new YarnConfiguration(csConf); + CapacityScheduler cs = new CapacityScheduler(); + + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); + cs.setConf(conf); + cs.setRMContext(rmContext); + cs.init(conf); + cs.start(); + + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + conf.set(CapacitySchedulerConfiguration.ENABLE_QUEUE_MAPPING_OVERRIDE, + "true"); + + // configuration parsing tests - negative test cases + checkInvalidQMapping(conf, cs, "x:a:b", "invalid specifier"); + checkInvalidQMapping(conf, cs, "u:a", "no queue specified"); + checkInvalidQMapping(conf, cs, "g:a", "no queue specified"); + checkInvalidQMapping(conf, cs, "u:a:b,g:a", + "multiple mappings with invalid mapping"); + checkInvalidQMapping(conf, cs, "u:a:b,g:a:d:e", "too many path segments"); + checkInvalidQMapping(conf, cs, "u::", "empty source and queue"); + checkInvalidQMapping(conf, cs, "u:", "missing source missing queue"); + checkInvalidQMapping(conf, cs, "u:a:", "empty source missing q"); + + // simple base case for mapping user to queue + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "u:a:" + Q1); + cs.reinitialize(conf, null); + checkQMapping("a", Q1, cs); + + // group mapping test + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "g:agroup:" + Q1); + cs.reinitialize(conf, null); + checkQMapping("a", Q1, cs); + + // %user tests + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "u:%user:" + Q2); + cs.reinitialize(conf, null); + checkQMapping("a", Q2, cs); + + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "u:%user:%user"); + cs.reinitialize(conf, null); + checkQMapping("a", "a", cs); + + // %primary_group tests + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, + "u:%user:%primary_group"); + cs.reinitialize(conf, null); + checkQMapping("a", "agroup", cs); + + // non-primary group mapping + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, + "g:asubgroup1:" + Q1); + cs.reinitialize(conf, null); + checkQMapping("a", Q1, cs); + + // space trimming + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1); + cs.reinitialize(conf, null); + checkQMapping("a", Q1, cs); + + csConf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + conf = new YarnConfiguration(csConf); + + resourceManager = new MockRM(csConf); + resourceManager.start(); + + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + conf.set(CapacitySchedulerConfiguration.ENABLE_QUEUE_MAPPING_OVERRIDE, + "true"); + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "u:user:" + Q1); + resourceManager.getResourceScheduler().reinitialize(conf, null); + + // ensure that if the user specifies a Q that is still overriden + checkAppQueue(resourceManager, "user", Q2, Q1); + + // toggle admin override and retry + conf.setBoolean( + CapacitySchedulerConfiguration.ENABLE_QUEUE_MAPPING_OVERRIDE, + false); + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "u:user:" + Q1); + setupQueueConfiguration(csConf); + resourceManager.getResourceScheduler().reinitialize(conf, null); + + checkAppQueue(resourceManager, "user", Q2, Q2); + + // ensure that if a user does not specify a Q, the user mapping is used + checkAppQueue(resourceManager, "user", null, Q1); + + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "g:usergroup:" + Q2); + setupQueueConfiguration(csConf); + resourceManager.getResourceScheduler().reinitialize(conf, null); + + // ensure that if a user does not specify a Q, the group mapping is used + checkAppQueue(resourceManager, "user", null, Q2); + + // if the mapping specifies a queue that does not exist, the job is rejected + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, + "u:user:non_existent_queue"); + setupQueueConfiguration(csConf); + + boolean fail = false; + try { + resourceManager.getResourceScheduler().reinitialize(conf, null); + } + catch (IOException ioex) { + fail = true; + } + Assert.assertTrue("queue initialization failed for non-existent q", fail); + resourceManager.stop(); + } + + private void checkAppQueue(MockRM resourceManager, String user, + String submissionQueue, String expected) + throws Exception { + RMApp app = resourceManager.submitApp(200, "name", user, + new HashMap(), false, submissionQueue, -1, + null, "MAPREDUCE", false); + RMAppState expectedState = expected.isEmpty() ? RMAppState.FAILED + : RMAppState.ACCEPTED; + resourceManager.waitForState(app.getApplicationId(), expectedState); + // get scheduler app + CapacityScheduler cs = (CapacityScheduler) + resourceManager.getResourceScheduler(); + SchedulerApplication schedulerApp = + cs.getSchedulerApplications().get(app.getApplicationId()); + String queue = ""; + if (schedulerApp != null) { + queue = schedulerApp.getQueue().getQueueName(); + } + Assert.assertTrue("expected " + expected + " actual " + queue, + expected.equals(queue)); + Assert.assertEquals(expected, app.getQueue()); + } + + private void checkInvalidQMapping(YarnConfiguration conf, + CapacityScheduler cs, + String mapping, String reason) + throws IOException { + boolean fail = false; + try { + conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, mapping); + cs.reinitialize(conf, null); + } catch (IOException ex) { + fail = true; + } + Assert.assertTrue("invalid mapping did not throw exception for " + reason, + fail); + } + + private void checkQMapping(String user, String expected, CapacityScheduler cs) + throws IOException { + String actual = cs.getMappedQueueForTest(user); + Assert.assertTrue("expected " + expected + " actual " + actual, + expected.equals(actual)); + } +} From 1c77c8041009196b03b2e27106bc6497e865e2a8 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 18 Aug 2014 15:34:58 +0000 Subject: [PATCH 59/67] HDFS-6783. Addendum patch to fix test failures. (Contributed by Yi Liu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618635 13f79535-47bb-0310-9956-ffa450edef68 --- .../CacheReplicationMonitor.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index cfd6333469b..fa5083ca6e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -286,19 +286,19 @@ public class CacheReplicationMonitor extends Thread implements Closeable { private void rescan() throws InterruptedException { scannedDirectives = 0; scannedBlocks = 0; - namesystem.writeLock(); try { - lock.lock(); - if (shutdown) { - throw new InterruptedException("CacheReplicationMonitor was " + - "shut down."); + namesystem.writeLock(); + try { + lock.lock(); + if (shutdown) { + throw new InterruptedException("CacheReplicationMonitor was " + + "shut down."); + } + curScanCount = completedScanCount + 1; + } finally { + lock.unlock(); } - curScanCount = completedScanCount + 1; - } - finally { - lock.unlock(); - } - try { + resetStatistics(); rescanCacheDirectives(); rescanCachedBlockMap(); From 08d6201b344cf7a7b4353774f06fb6e6b94541f0 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Mon, 18 Aug 2014 16:13:56 +0000 Subject: [PATCH 60/67] MAPREDUCE-6033. Updated access check for displaying job information (Yu Gao via Eric Yang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618648 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 2 +- .../hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java | 10 +++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 62b4e525644..142056c8260 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -308,6 +308,9 @@ Release 2.5.0 - UNRELEASED BUG FIXES + MAPREDUCE-6033. Updated access check for displaying job information + (Yu Gao via Eric Yang) + MAPREDUCE-5759. Remove unnecessary conf load in Limits (Sandy Ryza) MAPREDUCE-5014. Extend Distcp to accept a custom CopyListing. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 19dbb05c714..10c93f83e76 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -730,7 +730,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, if (jobACL == null) { return true; } - return aclsManager.checkAccess(callerUGI, jobOperation, username, jobACL); + return aclsManager.checkAccess(callerUGI, jobOperation, userName, jobACL); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java index 65352b75ad0..cae9663e8cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java @@ -536,7 +536,7 @@ public class TestJobImpl { // Verify access JobImpl job1 = new JobImpl(jobId, null, conf1, null, null, null, null, null, - null, null, null, true, null, 0, null, null, null, null); + null, null, null, true, user1, 0, null, null, null, null); Assert.assertTrue(job1.checkAccess(ugi1, JobACL.VIEW_JOB)); Assert.assertFalse(job1.checkAccess(ugi2, JobACL.VIEW_JOB)); @@ -547,7 +547,7 @@ public class TestJobImpl { // Verify access JobImpl job2 = new JobImpl(jobId, null, conf2, null, null, null, null, null, - null, null, null, true, null, 0, null, null, null, null); + null, null, null, true, user1, 0, null, null, null, null); Assert.assertTrue(job2.checkAccess(ugi1, JobACL.VIEW_JOB)); Assert.assertTrue(job2.checkAccess(ugi2, JobACL.VIEW_JOB)); @@ -558,7 +558,7 @@ public class TestJobImpl { // Verify access JobImpl job3 = new JobImpl(jobId, null, conf3, null, null, null, null, null, - null, null, null, true, null, 0, null, null, null, null); + null, null, null, true, user1, 0, null, null, null, null); Assert.assertTrue(job3.checkAccess(ugi1, JobACL.VIEW_JOB)); Assert.assertTrue(job3.checkAccess(ugi2, JobACL.VIEW_JOB)); @@ -569,7 +569,7 @@ public class TestJobImpl { // Verify access JobImpl job4 = new JobImpl(jobId, null, conf4, null, null, null, null, null, - null, null, null, true, null, 0, null, null, null, null); + null, null, null, true, user1, 0, null, null, null, null); Assert.assertTrue(job4.checkAccess(ugi1, JobACL.VIEW_JOB)); Assert.assertTrue(job4.checkAccess(ugi2, JobACL.VIEW_JOB)); @@ -580,7 +580,7 @@ public class TestJobImpl { // Verify access JobImpl job5 = new JobImpl(jobId, null, conf5, null, null, null, null, null, - null, null, null, true, null, 0, null, null, null, null); + null, null, null, true, user1, 0, null, null, null, null); Assert.assertTrue(job5.checkAccess(ugi1, null)); Assert.assertTrue(job5.checkAccess(ugi2, null)); } From 4a978bff3e8abc86850f89b5c3acffd522c56f80 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Mon, 18 Aug 2014 16:31:38 +0000 Subject: [PATCH 61/67] HADOOP-10059. RPC authentication and authorization metrics overflow to negative values on busy clusters. Contributed by Tsuyoshi OZAWA and Akira AJISAKA git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618659 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++++ .../java/org/apache/hadoop/ipc/metrics/RpcMetrics.java | 8 ++++---- .../src/test/java/org/apache/hadoop/ipc/TestRPC.java | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fb0578f7c50..fff78128619 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -605,6 +605,10 @@ Release 2.6.0 - UNRELEASED HADOOP-10964. Small fix for NetworkTopologyWithNodeGroup#sortByDistance. (Yi Liu via wang) + HADOOP-10059. RPC authentication and authorization metrics overflow to + negative values on busy clusters (Tsuyoshi OZAWA and Akira AJISAKA + via jlowe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index 4b2269d6b6a..5eba44a58f6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -88,13 +88,13 @@ public class RpcMetrics { @Metric("Processsing time") MutableRate rpcProcessingTime; MutableQuantiles[] rpcProcessingTimeMillisQuantiles; @Metric("Number of authentication failures") - MutableCounterInt rpcAuthenticationFailures; + MutableCounterLong rpcAuthenticationFailures; @Metric("Number of authentication successes") - MutableCounterInt rpcAuthenticationSuccesses; + MutableCounterLong rpcAuthenticationSuccesses; @Metric("Number of authorization failures") - MutableCounterInt rpcAuthorizationFailures; + MutableCounterLong rpcAuthorizationFailures; @Metric("Number of authorization sucesses") - MutableCounterInt rpcAuthorizationSuccesses; + MutableCounterLong rpcAuthorizationSuccesses; @Metric("Number of open connections") public int numOpenConnections() { return server.getNumOpenConnections(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index dfbc91c43a6..f0e389ff5de 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -583,14 +583,14 @@ public class TestRPC { } MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name()); if (expectFailure) { - assertCounter("RpcAuthorizationFailures", 1, rb); + assertCounter("RpcAuthorizationFailures", 1L, rb); } else { - assertCounter("RpcAuthorizationSuccesses", 1, rb); + assertCounter("RpcAuthorizationSuccesses", 1L, rb); } //since we don't have authentication turned ON, we should see // 0 for the authentication successes and 0 for failure - assertCounter("RpcAuthenticationFailures", 0, rb); - assertCounter("RpcAuthenticationSuccesses", 0, rb); + assertCounter("RpcAuthenticationFailures", 0L, rb); + assertCounter("RpcAuthenticationSuccesses", 0L, rb); } } From f8e871d01b851cd5d8c57dd7e364b3e787521765 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Mon, 18 Aug 2014 17:57:48 +0000 Subject: [PATCH 62/67] MAPREDUCE-6024. Shortened the time when Fetcher is stuck in retrying before concluding the failure by configuration. Contributed by Yunjiong Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618677 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../mapreduce/v2/app/job/impl/JobImpl.java | 28 +++++++++++---- .../apache/hadoop/mapreduce/MRJobConfig.java | 8 +++++ .../hadoop/mapreduce/task/reduce/Fetcher.java | 2 ++ .../task/reduce/ShuffleSchedulerImpl.java | 34 +++++++++++++------ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 142056c8260..ddec273dcb1 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -227,6 +227,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6032. Made MR jobs write job history files on the default FS when the current context's FS is different. (Benjamin Zhitomirsky via zjshen) + MAPREDUCE-6024. Shortened the time when Fetcher is stuck in retrying before + concluding the failure by configuration. (Yunjiong Zhao via zjshen) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 10c93f83e76..c1bc17df97c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -148,10 +148,10 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private static final Log LOG = LogFactory.getLog(JobImpl.class); //The maximum fraction of fetch failures allowed for a map - private static final double MAX_ALLOWED_FETCH_FAILURES_FRACTION = 0.5; - - // Maximum no. of fetch-failure notifications after which map task is failed - private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3; + private float maxAllowedFetchFailuresFraction; + + //Maximum no. of fetch-failure notifications after which map task is failed + private int maxFetchFailuresNotifications; public static final String JOB_KILLED_DIAG = "Job received Kill while in RUNNING state."; @@ -704,6 +704,13 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, if(forcedDiagnostic != null) { this.diagnostics.add(forcedDiagnostic); } + + this.maxAllowedFetchFailuresFraction = conf.getFloat( + MRJobConfig.MAX_ALLOWED_FETCH_FAILURES_FRACTION, + MRJobConfig.DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION); + this.maxFetchFailuresNotifications = conf.getInt( + MRJobConfig.MAX_FETCH_FAILURES_NOTIFICATIONS, + MRJobConfig.DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS); } protected StateMachine getStateMachine() { @@ -1900,9 +1907,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, float failureRate = shufflingReduceTasks == 0 ? 1.0f : (float) fetchFailures / shufflingReduceTasks; // declare faulty if fetch-failures >= max-allowed-failures - boolean isMapFaulty = - (failureRate >= MAX_ALLOWED_FETCH_FAILURES_FRACTION); - if (fetchFailures >= MAX_FETCH_FAILURES_NOTIFICATIONS && isMapFaulty) { + if (fetchFailures >= job.getMaxFetchFailuresNotifications() + && failureRate >= job.getMaxAllowedFetchFailuresFraction()) { LOG.info("Too many fetch-failures for output of task attempt: " + mapId + " ... raising fetch failure to map"); job.eventHandler.handle(new TaskAttemptEvent(mapId, @@ -2185,4 +2191,12 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; } + + public float getMaxAllowedFetchFailuresFraction() { + return maxAllowedFetchFailuresFraction; + } + + public int getMaxFetchFailuresNotifications() { + return maxFetchFailuresNotifications; + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 0a586967d01..aef84c0ccad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -293,11 +293,19 @@ public interface MRJobConfig { public static final String SHUFFLE_READ_TIMEOUT = "mapreduce.reduce.shuffle.read.timeout"; public static final String SHUFFLE_FETCH_FAILURES = "mapreduce.reduce.shuffle.maxfetchfailures"; + public static final String MAX_ALLOWED_FETCH_FAILURES_FRACTION = "mapreduce.reduce.shuffle.max-fetch-failures-fraction"; + public static final float DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION = 0.5f; + + public static final String MAX_FETCH_FAILURES_NOTIFICATIONS = "mapreduce.reduce.shuffle.max-fetch-failures-notifications"; + public static final int DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS = 3; public static final String SHUFFLE_NOTIFY_READERROR = "mapreduce.reduce.shuffle.notify.readerror"; public static final String MAX_SHUFFLE_FETCH_RETRY_DELAY = "mapreduce.reduce.shuffle.retry-delay.max.ms"; public static final long DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY = 60000; + + public static final String MAX_SHUFFLE_FETCH_HOST_FAILURES = "mapreduce.reduce.shuffle.max-host-failures"; + public static final int DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES = 5; public static final String REDUCE_SKIP_INCR_PROC_COUNT = "mapreduce.reduce.skip.proc-count.auto-incr"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java index 00d4764e665..94966b91560 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java @@ -319,6 +319,7 @@ class Fetcher extends Thread { // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host + scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } @@ -343,6 +344,7 @@ class Fetcher extends Thread { if(failedTasks != null && failedTasks.length > 0) { LOG.warn("copyMapOutput failed for tasks "+Arrays.toString(failedTasks)); + scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: failedTasks) { scheduler.copyFailed(left, host, true, false); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java index 6f9b222bdcf..63f326632ef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java @@ -18,7 +18,6 @@ package org.apache.hadoop.mapreduce.task.reduce; import java.io.IOException; - import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; @@ -101,6 +100,7 @@ public class ShuffleSchedulerImpl implements ShuffleScheduler { private final boolean reportReadErrorImmediately; private long maxDelay = MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY; + private int maxHostFailures; public ShuffleSchedulerImpl(JobConf job, TaskStatus status, TaskAttemptID reduceId, @@ -132,6 +132,9 @@ public class ShuffleSchedulerImpl implements ShuffleScheduler { this.maxDelay = job.getLong(MRJobConfig.MAX_SHUFFLE_FETCH_RETRY_DELAY, MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY); + this.maxHostFailures = job.getInt( + MRJobConfig.MAX_SHUFFLE_FETCH_HOST_FAILURES, + MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES); } @Override @@ -213,9 +216,18 @@ public class ShuffleSchedulerImpl implements ShuffleScheduler { progress.setStatus("copy(" + mapsDone + " of " + totalMaps + " at " + mbpsFormat.format(transferRate) + " MB/s)"); } + + public synchronized void hostFailed(String hostname) { + if (hostFailures.containsKey(hostname)) { + IntWritable x = hostFailures.get(hostname); + x.set(x.get() + 1); + } else { + hostFailures.put(hostname, new IntWritable(1)); + } + } public synchronized void copyFailed(TaskAttemptID mapId, MapHost host, - boolean readError, boolean connectExcpt) { + boolean readError, boolean connectExcpt) { host.penalize(); int failures = 1; if (failureCounts.containsKey(mapId)) { @@ -226,12 +238,9 @@ public class ShuffleSchedulerImpl implements ShuffleScheduler { failureCounts.put(mapId, new IntWritable(1)); } String hostname = host.getHostName(); - if (hostFailures.containsKey(hostname)) { - IntWritable x = hostFailures.get(hostname); - x.set(x.get() + 1); - } else { - hostFailures.put(hostname, new IntWritable(1)); - } + //report failure if already retried maxHostFailures times + boolean hostFail = hostFailures.get(hostname).get() > getMaxHostFailures() ? true : false; + if (failures >= abortFailureLimit) { try { throw new IOException(failures + " failures downloading " + mapId); @@ -240,7 +249,7 @@ public class ShuffleSchedulerImpl implements ShuffleScheduler { } } - checkAndInformJobTracker(failures, mapId, readError, connectExcpt); + checkAndInformJobTracker(failures, mapId, readError, connectExcpt, hostFail); checkReducerHealth(); @@ -270,9 +279,9 @@ public class ShuffleSchedulerImpl implements ShuffleScheduler { // after every 'maxFetchFailuresBeforeReporting' failures private void checkAndInformJobTracker( int failures, TaskAttemptID mapId, boolean readError, - boolean connectExcpt) { + boolean connectExcpt, boolean hostFailed) { if (connectExcpt || (reportReadErrorImmediately && readError) - || ((failures % maxFetchFailuresBeforeReporting) == 0)) { + || ((failures % maxFetchFailuresBeforeReporting) == 0) || hostFailed) { LOG.info("Reporting fetch failure for " + mapId + " to jobtracker."); status.addFetchFailedMap((org.apache.hadoop.mapred.TaskAttemptID) mapId); } @@ -507,4 +516,7 @@ public class ShuffleSchedulerImpl implements ShuffleScheduler { referee.join(); } + public int getMaxHostFailures() { + return maxHostFailures; + } } From e446497cd1e9b892659e9cfb79d37f33a4e39de0 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Mon, 18 Aug 2014 18:02:37 +0000 Subject: [PATCH 63/67] HDFS-6561. org.apache.hadoop.util.DataChecksum should support native checksumming (James Thomas via Colin Patrick McCabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618680 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/util/DataChecksum.java | 14 +- .../org/apache/hadoop/util/NativeCrc32.java | 33 +++- .../src/org/apache/hadoop/util/NativeCrc32.c | 30 ++-- .../src/org/apache/hadoop/util/bulk_crc32.c | 60 +++---- .../src/org/apache/hadoop/util/bulk_crc32.h | 49 ++---- .../org/apache/hadoop/util/test_bulk_crc32.c | 6 +- .../apache/hadoop/util/TestDataChecksum.java | 154 ++++++++++++------ 8 files changed, 197 insertions(+), 152 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fff78128619..69aec036ed8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -527,6 +527,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10335. An ip whilelist based implementation to resolve Sasl properties per connection. (Benoy Antony via Arpit Agarwal) + HDFS-6561. org.apache.hadoop.util.DataChecksum should support native + checksumming (James Thomas via Colin Patrick McCabe) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index 27984b48c7e..1636af68a36 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -390,6 +390,12 @@ public class DataChecksum implements Checksum { checksums.array(), checksums.arrayOffset() + checksums.position()); return; } + + if (NativeCrc32.isAvailable()) { + NativeCrc32.calculateChunkedSums(bytesPerChecksum, type.id, + checksums, data); + return; + } data.mark(); checksums.mark(); @@ -412,10 +418,16 @@ public class DataChecksum implements Checksum { * Implementation of chunked calculation specifically on byte arrays. This * is to avoid the copy when dealing with ByteBuffers that have array backing. */ - private void calculateChunkedSums( + public void calculateChunkedSums( byte[] data, int dataOffset, int dataLength, byte[] sums, int sumsOffset) { + if (NativeCrc32.isAvailable()) { + NativeCrc32.calculateChunkedSumsByteArray(bytesPerChecksum, type.id, + sums, sumsOffset, data, dataOffset, dataLength); + return; + } + int remaining = dataLength; while (remaining > 0) { int n = Math.min(remaining, bytesPerChecksum); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java index 26a5e9b10dd..2f21ae1a03d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java @@ -54,33 +54,50 @@ class NativeCrc32 { public static void verifyChunkedSums(int bytesPerSum, int checksumType, ByteBuffer sums, ByteBuffer data, String fileName, long basePos) throws ChecksumException { - nativeVerifyChunkedSums(bytesPerSum, checksumType, + nativeComputeChunkedSums(bytesPerSum, checksumType, sums, sums.position(), data, data.position(), data.remaining(), - fileName, basePos); + fileName, basePos, true); } public static void verifyChunkedSumsByteArray(int bytesPerSum, int checksumType, byte[] sums, int sumsOffset, byte[] data, int dataOffset, int dataLength, String fileName, long basePos) throws ChecksumException { - nativeVerifyChunkedSumsByteArray(bytesPerSum, checksumType, + nativeComputeChunkedSumsByteArray(bytesPerSum, checksumType, sums, sumsOffset, data, dataOffset, dataLength, - fileName, basePos); + fileName, basePos, true); + } + + public static void calculateChunkedSums(int bytesPerSum, int checksumType, + ByteBuffer sums, ByteBuffer data) { + nativeComputeChunkedSums(bytesPerSum, checksumType, + sums, sums.position(), + data, data.position(), data.remaining(), + "", 0, false); + } + + public static void calculateChunkedSumsByteArray(int bytesPerSum, + int checksumType, byte[] sums, int sumsOffset, byte[] data, + int dataOffset, int dataLength) { + nativeComputeChunkedSumsByteArray(bytesPerSum, checksumType, + sums, sumsOffset, + data, dataOffset, dataLength, + "", 0, false); } - private static native void nativeVerifyChunkedSums( + private static native void nativeComputeChunkedSums( int bytesPerSum, int checksumType, ByteBuffer sums, int sumsOffset, ByteBuffer data, int dataOffset, int dataLength, - String fileName, long basePos); + String fileName, long basePos, boolean verify); - private static native void nativeVerifyChunkedSumsByteArray( + private static native void nativeComputeChunkedSumsByteArray( int bytesPerSum, int checksumType, byte[] sums, int sumsOffset, byte[] data, int dataOffset, int dataLength, - String fileName, long basePos); + String fileName, long basePos, boolean verify); // Copy the constants over from DataChecksum so that javah will pick them up // and make them available in the native code header. diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c index 4bd69bc41d7..899c59f9809 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -117,12 +117,12 @@ static int convert_java_crc_type(JNIEnv *env, jint crc_type) { } } -JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunkedSums +JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSums (JNIEnv *env, jclass clazz, jint bytes_per_checksum, jint j_crc_type, jobject j_sums, jint sums_offset, jobject j_data, jint data_offset, jint data_len, - jstring j_filename, jlong base_pos) + jstring j_filename, jlong base_pos, jboolean verify) { uint8_t *sums_addr; uint8_t *data_addr; @@ -166,27 +166,27 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk if (crc_type == -1) return; // exception already thrown // Setup complete. Actually verify checksums. - ret = bulk_verify_crc(data, data_len, sums, crc_type, - bytes_per_checksum, &error_data); - if (likely(ret == CHECKSUMS_VALID)) { + ret = bulk_crc(data, data_len, sums, crc_type, + bytes_per_checksum, verify ? &error_data : NULL); + if (likely(verify && ret == CHECKSUMS_VALID || !verify && ret == 0)) { return; - } else if (unlikely(ret == INVALID_CHECKSUM_DETECTED)) { + } else if (unlikely(verify && ret == INVALID_CHECKSUM_DETECTED)) { long pos = base_pos + (error_data.bad_data - data); throw_checksum_exception( env, error_data.got_crc, error_data.expected_crc, j_filename, pos); } else { THROW(env, "java/lang/AssertionError", - "Bad response code from native bulk_verify_crc"); + "Bad response code from native bulk_crc"); } } -JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunkedSumsByteArray +JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSumsByteArray (JNIEnv *env, jclass clazz, jint bytes_per_checksum, jint j_crc_type, jarray j_sums, jint sums_offset, jarray j_data, jint data_offset, jint data_len, - jstring j_filename, jlong base_pos) + jstring j_filename, jlong base_pos, jboolean verify) { uint8_t *sums_addr; uint8_t *data_addr; @@ -237,21 +237,21 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk data = data_addr + data_offset + checksumNum * bytes_per_checksum; // Setup complete. Actually verify checksums. - ret = bulk_verify_crc(data, MIN(numChecksumsPerIter * bytes_per_checksum, - data_len - checksumNum * bytes_per_checksum), - sums, crc_type, bytes_per_checksum, &error_data); + ret = bulk_crc(data, MIN(numChecksumsPerIter * bytes_per_checksum, + data_len - checksumNum * bytes_per_checksum), + sums, crc_type, bytes_per_checksum, verify ? &error_data : NULL); (*env)->ReleasePrimitiveArrayCritical(env, j_data, data_addr, 0); (*env)->ReleasePrimitiveArrayCritical(env, j_sums, sums_addr, 0); - if (unlikely(ret == INVALID_CHECKSUM_DETECTED)) { + if (unlikely(verify && ret == INVALID_CHECKSUM_DETECTED)) { long pos = base_pos + (error_data.bad_data - data) + checksumNum * bytes_per_checksum; throw_checksum_exception( env, error_data.got_crc, error_data.expected_crc, j_filename, pos); return; - } else if (unlikely(ret != CHECKSUMS_VALID)) { + } else if (unlikely(verify && ret != CHECKSUMS_VALID || !verify && ret != 0)) { THROW(env, "java/lang/AssertionError", - "Bad response code from native bulk_verify_crc"); + "Bad response code from native bulk_crc"); return; } checksumNum += numChecksumsPerIter; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c index 4f02eedd6a7..c7efb8db95c 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c @@ -55,40 +55,23 @@ static void pipelined_crc32c(uint32_t *crc1, uint32_t *crc2, uint32_t *crc3, con static int cached_cpu_supports_crc32; // initialized by constructor below static uint32_t crc32c_hardware(uint32_t crc, const uint8_t* data, size_t length); -int bulk_calculate_crc(const uint8_t *data, size_t data_len, - uint32_t *sums, int checksum_type, - int bytes_per_checksum) { - uint32_t crc; - crc_update_func_t crc_update_func; - - switch (checksum_type) { - case CRC32_ZLIB_POLYNOMIAL: - crc_update_func = crc32_zlib_sb8; - break; - case CRC32C_POLYNOMIAL: - crc_update_func = crc32c_sb8; - break; - default: - return -EINVAL; - break; +static inline int store_or_verify(uint32_t *sums, uint32_t crc, + int is_verify) { + if (!is_verify) { + *sums = crc; + return 1; + } else { + return crc == *sums; } - while (likely(data_len > 0)) { - int len = likely(data_len >= bytes_per_checksum) ? bytes_per_checksum : data_len; - crc = CRC_INITIAL_VAL; - crc = crc_update_func(crc, data, len); - *sums = ntohl(crc_val(crc)); - data += len; - data_len -= len; - sums++; - } - return 0; } -int bulk_verify_crc(const uint8_t *data, size_t data_len, - const uint32_t *sums, int checksum_type, +int bulk_crc(const uint8_t *data, size_t data_len, + uint32_t *sums, int checksum_type, int bytes_per_checksum, crc32_error_t *error_info) { + int is_verify = error_info != NULL; + #ifdef USE_PIPELINED uint32_t crc1, crc2, crc3; int n_blocks = data_len / bytes_per_checksum; @@ -112,7 +95,7 @@ int bulk_verify_crc(const uint8_t *data, size_t data_len, } break; default: - return INVALID_CHECKSUM_TYPE; + return is_verify ? INVALID_CHECKSUM_TYPE : -EINVAL; } #ifdef USE_PIPELINED @@ -122,16 +105,15 @@ int bulk_verify_crc(const uint8_t *data, size_t data_len, crc1 = crc2 = crc3 = CRC_INITIAL_VAL; pipelined_crc32c(&crc1, &crc2, &crc3, data, bytes_per_checksum, 3); - crc = ntohl(crc_val(crc1)); - if ((crc = ntohl(crc_val(crc1))) != *sums) + if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc1))), is_verify))) goto return_crc_error; sums++; data += bytes_per_checksum; - if ((crc = ntohl(crc_val(crc2))) != *sums) + if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc2))), is_verify))) goto return_crc_error; sums++; data += bytes_per_checksum; - if ((crc = ntohl(crc_val(crc3))) != *sums) + if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc3))), is_verify))) goto return_crc_error; sums++; data += bytes_per_checksum; @@ -143,12 +125,12 @@ int bulk_verify_crc(const uint8_t *data, size_t data_len, crc1 = crc2 = crc3 = CRC_INITIAL_VAL; pipelined_crc32c(&crc1, &crc2, &crc3, data, bytes_per_checksum, n_blocks); - if ((crc = ntohl(crc_val(crc1))) != *sums) + if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc1))), is_verify))) goto return_crc_error; data += bytes_per_checksum; sums++; if (n_blocks == 2) { - if ((crc = ntohl(crc_val(crc2))) != *sums) + if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc2))), is_verify))) goto return_crc_error; sums++; data += bytes_per_checksum; @@ -160,10 +142,10 @@ int bulk_verify_crc(const uint8_t *data, size_t data_len, crc1 = crc2 = crc3 = CRC_INITIAL_VAL; pipelined_crc32c(&crc1, &crc2, &crc3, data, remainder, 1); - if ((crc = ntohl(crc_val(crc1))) != *sums) + if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc1))), is_verify))) goto return_crc_error; } - return CHECKSUMS_VALID; + return is_verify ? CHECKSUMS_VALID : 0; } #endif @@ -172,14 +154,14 @@ int bulk_verify_crc(const uint8_t *data, size_t data_len, crc = CRC_INITIAL_VAL; crc = crc_update_func(crc, data, len); crc = ntohl(crc_val(crc)); - if (unlikely(crc != *sums)) { + if (unlikely(!store_or_verify(sums, crc, is_verify))) { goto return_crc_error; } data += len; data_len -= len; sums++; } - return CHECKSUMS_VALID; + return is_verify ? CHECKSUMS_VALID : 0; return_crc_error: if (error_info != NULL) { diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h index fce5358d646..b38a65acc6b 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h @@ -42,49 +42,32 @@ typedef struct crc32_error { /** - * Verify a buffer of data which is checksummed in chunks - * of bytes_per_checksum bytes. The checksums are each 32 bits - * and are stored in sequential indexes of the 'sums' array. + * Either calculates checksums for or verifies a buffer of data. + * Checksums performed in chunks of bytes_per_checksum bytes. The checksums + * are each 32 bits and are stored in sequential indexes of the 'sums' array. + * Verification is done (sums is assumed to already contain the checksums) + * if error_info is non-null; otherwise calculation is done and checksums + * are stored into sums. * * @param data The data to checksum * @param dataLen Length of the data buffer - * @param sums (out param) buffer to write checksums into. - * It must contain at least dataLen * 4 bytes. + * @param sums (out param) buffer to write checksums into or + * where checksums are already stored. + * It must contain at least + * ((dataLen - 1) / bytes_per_checksum + 1) * 4 bytes. * @param checksum_type One of the CRC32 algorithm constants defined * above * @param bytes_per_checksum How many bytes of data to process per checksum. - * @param error_info If non-NULL, will be filled in if an error - * is detected + * @param error_info If non-NULL, verification will be performed and + * it will be filled in if an error + * is detected. Otherwise calculation is performed. * * @return 0 for success, non-zero for an error, result codes - * for which are defined above + * for verification are defined above */ -extern int bulk_verify_crc(const uint8_t *data, size_t data_len, - const uint32_t *sums, int checksum_type, +extern int bulk_crc(const uint8_t *data, size_t data_len, + uint32_t *sums, int checksum_type, int bytes_per_checksum, crc32_error_t *error_info); -/** - * Calculate checksums for some data. - * - * The checksums are each 32 bits and are stored in sequential indexes of the - * 'sums' array. - * - * This function is not (yet) optimized. It is provided for testing purposes - * only. - * - * @param data The data to checksum - * @param dataLen Length of the data buffer - * @param sums (out param) buffer to write checksums into. - * It must contain at least dataLen * 4 bytes. - * @param checksum_type One of the CRC32 algorithm constants defined - * above - * @param bytesPerChecksum How many bytes of data to process per checksum. - * - * @return 0 for success, non-zero for an error - */ -int bulk_calculate_crc(const uint8_t *data, size_t data_len, - uint32_t *sums, int checksum_type, - int bytes_per_checksum); - #endif diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c index c962337830e..5a8c9f2d855 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c @@ -48,9 +48,9 @@ static int testBulkVerifyCrc(int dataLen, int crcType, int bytesPerChecksum) sums = calloc(sizeof(uint32_t), (dataLen + bytesPerChecksum - 1) / bytesPerChecksum); - EXPECT_ZERO(bulk_calculate_crc(data, dataLen, sums, crcType, - bytesPerChecksum)); - EXPECT_ZERO(bulk_verify_crc(data, dataLen, sums, crcType, + EXPECT_ZERO(bulk_crc(data, dataLen, sums, crcType, + bytesPerChecksum, NULL)); + EXPECT_ZERO(bulk_crc(data, dataLen, sums, crcType, bytesPerChecksum, &errorData)); free(data); free(sums); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java index 1e523da6948..34fc32aa08f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java @@ -19,6 +19,9 @@ package org.apache.hadoop.util; import java.nio.ByteBuffer; import java.util.Random; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Stopwatch; import org.apache.hadoop.fs.ChecksumException; import org.junit.Test; @@ -53,68 +56,113 @@ public class TestDataChecksum { } } } - - private void doBulkTest(DataChecksum checksum, int dataLength, - boolean useDirect) throws Exception { - System.err.println("Testing bulk checksums of length " + - dataLength + " with " + - (useDirect ? "direct" : "array-backed") + " buffers"); - int numSums = (dataLength - 1)/checksum.getBytesPerChecksum() + 1; - int sumsLength = numSums * checksum.getChecksumSize(); - - byte data[] = new byte[dataLength + - DATA_OFFSET_IN_BUFFER + - DATA_TRAILER_IN_BUFFER]; - new Random().nextBytes(data); - ByteBuffer dataBuf = ByteBuffer.wrap( + + private static class Harness { + final DataChecksum checksum; + final int dataLength, sumsLength, numSums; + ByteBuffer dataBuf, checksumBuf; + + Harness(DataChecksum checksum, int dataLength, boolean useDirect) { + this.checksum = checksum; + this.dataLength = dataLength; + + numSums = (dataLength - 1)/checksum.getBytesPerChecksum() + 1; + sumsLength = numSums * checksum.getChecksumSize(); + + byte data[] = new byte[dataLength + + DATA_OFFSET_IN_BUFFER + + DATA_TRAILER_IN_BUFFER]; + new Random().nextBytes(data); + dataBuf = ByteBuffer.wrap( data, DATA_OFFSET_IN_BUFFER, dataLength); - byte checksums[] = new byte[SUMS_OFFSET_IN_BUFFER + sumsLength]; - ByteBuffer checksumBuf = ByteBuffer.wrap( + byte checksums[] = new byte[SUMS_OFFSET_IN_BUFFER + sumsLength]; + checksumBuf = ByteBuffer.wrap( checksums, SUMS_OFFSET_IN_BUFFER, sumsLength); - - // Swap out for direct buffers if requested. - if (useDirect) { - dataBuf = directify(dataBuf); - checksumBuf = directify(checksumBuf); - } - - // calculate real checksum, make sure it passes - checksum.calculateChunkedSums(dataBuf, checksumBuf); - checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); - // Change a byte in the header and in the trailer, make sure - // it doesn't affect checksum result - corruptBufferOffset(checksumBuf, 0); - checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); - corruptBufferOffset(dataBuf, 0); - dataBuf.limit(dataBuf.limit() + 1); - corruptBufferOffset(dataBuf, dataLength + DATA_OFFSET_IN_BUFFER); - dataBuf.limit(dataBuf.limit() - 1); - checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); - - // Make sure bad checksums fail - error at beginning of array - corruptBufferOffset(checksumBuf, SUMS_OFFSET_IN_BUFFER); - try { - checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); - fail("Did not throw on bad checksums"); - } catch (ChecksumException ce) { - assertEquals(0, ce.getPos()); + // Swap out for direct buffers if requested. + if (useDirect) { + dataBuf = directify(dataBuf); + checksumBuf = directify(checksumBuf); + } } - // Make sure bad checksums fail - error at end of array - uncorruptBufferOffset(checksumBuf, SUMS_OFFSET_IN_BUFFER); - corruptBufferOffset(checksumBuf, SUMS_OFFSET_IN_BUFFER + sumsLength - 1); - try { + void testCorrectness() throws ChecksumException { + // calculate real checksum, make sure it passes + checksum.calculateChunkedSums(dataBuf, checksumBuf); checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); - fail("Did not throw on bad checksums"); - } catch (ChecksumException ce) { - int expectedPos = checksum.getBytesPerChecksum() * (numSums - 1); - assertEquals(expectedPos, ce.getPos()); - assertTrue(ce.getMessage().contains("fake file")); + + // Change a byte in the header and in the trailer, make sure + // it doesn't affect checksum result + corruptBufferOffset(checksumBuf, 0); + checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); + corruptBufferOffset(dataBuf, 0); + dataBuf.limit(dataBuf.limit() + 1); + corruptBufferOffset(dataBuf, dataLength + DATA_OFFSET_IN_BUFFER); + dataBuf.limit(dataBuf.limit() - 1); + checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); + + // Make sure bad checksums fail - error at beginning of array + corruptBufferOffset(checksumBuf, SUMS_OFFSET_IN_BUFFER); + try { + checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); + fail("Did not throw on bad checksums"); + } catch (ChecksumException ce) { + assertEquals(0, ce.getPos()); + } + + // Make sure bad checksums fail - error at end of array + uncorruptBufferOffset(checksumBuf, SUMS_OFFSET_IN_BUFFER); + corruptBufferOffset(checksumBuf, SUMS_OFFSET_IN_BUFFER + sumsLength - 1); + try { + checksum.verifyChunkedSums(dataBuf, checksumBuf, "fake file", 0); + fail("Did not throw on bad checksums"); + } catch (ChecksumException ce) { + int expectedPos = checksum.getBytesPerChecksum() * (numSums - 1); + assertEquals(expectedPos, ce.getPos()); + assertTrue(ce.getMessage().contains("fake file")); + } } } - + + private void doBulkTest(DataChecksum checksum, int dataLength, + boolean useDirect) throws Exception { + System.err.println("Testing bulk checksums of length " + + dataLength + " with " + + (useDirect ? "direct" : "array-backed") + " buffers"); + + new Harness(checksum, dataLength, useDirect).testCorrectness(); + } + + /** + * Simple performance test for the "common case" checksum usage in HDFS: + * computing and verifying CRC32C with 512 byte chunking on native + * buffers. + */ + @Test + public void commonUsagePerfTest() throws Exception { + final int NUM_RUNS = 5; + final DataChecksum checksum = DataChecksum.newDataChecksum(DataChecksum.Type.CRC32C, 512); + final int dataLength = 512 * 1024 * 1024; + Harness h = new Harness(checksum, dataLength, true); + + for (int i = 0; i < NUM_RUNS; i++) { + Stopwatch s = new Stopwatch().start(); + // calculate real checksum, make sure it passes + checksum.calculateChunkedSums(h.dataBuf, h.checksumBuf); + s.stop(); + System.err.println("Calculate run #" + i + ": " + + s.elapsedTime(TimeUnit.MICROSECONDS) + "us"); + + s = new Stopwatch().start(); + // calculate real checksum, make sure it passes + checksum.verifyChunkedSums(h.dataBuf, h.checksumBuf, "fake file", 0); + s.stop(); + System.err.println("Verify run #" + i + ": " + + s.elapsedTime(TimeUnit.MICROSECONDS) + "us"); + } + } + @Test public void testEquality() { assertEquals( From 110cf46a0725798bd5ed1d225f7c01fbb685a718 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 18 Aug 2014 18:05:36 +0000 Subject: [PATCH 64/67] HADOOP-10973. Native Libraries Guide contains format error. (Contributed by Peter Klavins) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618682 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/site/apt/NativeLibraries.apt.vm | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 69aec036ed8..32507184956 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -612,6 +612,9 @@ Release 2.6.0 - UNRELEASED negative values on busy clusters (Tsuyoshi OZAWA and Akira AJISAKA via jlowe) + HADOOP-10973. Native Libraries Guide contains format error. (Peter Klavins + via Arpit Agarwal) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm index 070a7d3e568..ab06a7c61d4 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm @@ -56,10 +56,12 @@ Native Libraries Guide [[4]] Install the compression codec development packages (>zlib-1.2, >gzip-1.2): - + If you download the library, install one or more development + + * If you download the library, install one or more development packages - whichever compression codecs you want to use with your deployment. - + If you build the library, it is mandatory to install both + + * If you build the library, it is mandatory to install both development packages. [[5]] Check the runtime log files. From bd616552cce7e46d1cc27ad9aba38f96a1f4c29c Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 18 Aug 2014 18:08:42 +0000 Subject: [PATCH 65/67] HDFS-6825. Edit log corruption due to delayed block removal. Contributed by Yongjun Zhang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618684 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../BlockInfoUnderConstruction.java | 12 +- .../hdfs/server/namenode/FSNamesystem.java | 50 ++++++- .../hdfs/server/namenode/INodeDirectory.java | 2 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 32 +++++ .../TestCommitBlockSynchronization.java | 11 ++ .../hdfs/server/namenode/TestDeleteRace.java | 133 ++++++++++++++++++ .../namenode/ha/TestPipelinesFailover.java | 34 +---- 8 files changed, 235 insertions(+), 42 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e7013146cd1..cb80e33d303 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -511,6 +511,9 @@ Release 2.6.0 - UNRELEASED HDFS-6783. Fix HDFS CacheReplicationMonitor rescan logic. (Yi Liu and Colin Patrick McCabe via umamahesh) + HDFS-6825. Edit log corruption due to delayed block removal. + (Yongjun Zhang via wang) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java index 1161077f49d..dd3593f33bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java @@ -373,12 +373,14 @@ public class BlockInfoUnderConstruction extends BlockInfo { sb.append("{blockUCState=").append(blockUCState) .append(", primaryNodeIndex=").append(primaryNodeIndex) .append(", replicas=["); - Iterator iter = replicas.iterator(); - if (iter.hasNext()) { - iter.next().appendStringTo(sb); - while (iter.hasNext()) { - sb.append(", "); + if (replicas != null) { + Iterator iter = replicas.iterator(); + if (iter.hasNext()) { iter.next().appendStringTo(sb); + while (iter.hasNext()) { + sb.append(", "); + iter.next().appendStringTo(sb); + } } } sb.append("]}"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index b9cda6cefc9..0cb90bd2039 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -4300,7 +4300,30 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throw new IOException("Block (=" + lastblock + ") not found"); } } - INodeFile iFile = ((INode)storedBlock.getBlockCollection()).asFile(); + // + // The implementation of delete operation (see @deleteInternal method) + // first removes the file paths from namespace, and delays the removal + // of blocks to later time for better performance. When + // commitBlockSynchronization (this method) is called in between, the + // blockCollection of storedBlock could have been assigned to null by + // the delete operation, throw IOException here instead of NPE; if the + // file path is already removed from namespace by the delete operation, + // throw FileNotFoundException here, so not to proceed to the end of + // this method to add a CloseOp to the edit log for an already deleted + // file (See HDFS-6825). + // + BlockCollection blockCollection = storedBlock.getBlockCollection(); + if (blockCollection == null) { + throw new IOException("The blockCollection of " + storedBlock + + " is null, likely because the file owning this block was" + + " deleted and the block removal is delayed"); + } + INodeFile iFile = ((INode)blockCollection).asFile(); + if (isFileDeleted(iFile)) { + throw new FileNotFoundException("File not found: " + + iFile.getFullPathName() + ", likely due to delayed block" + + " removal"); + } if (!iFile.isUnderConstruction() || storedBlock.isComplete()) { if (LOG.isDebugEnabled()) { LOG.debug("Unexpected block (=" + lastblock @@ -6297,9 +6320,28 @@ public class FSNamesystem implements Namesystem, FSClusterStats, private boolean isFileDeleted(INodeFile file) { // Not in the inodeMap or in the snapshot but marked deleted. - if (dir.getInode(file.getId()) == null || - file.getParent() == null || (file.isWithSnapshot() && - file.getFileWithSnapshotFeature().isCurrentFileDeleted())) { + if (dir.getInode(file.getId()) == null) { + return true; + } + + // look at the path hierarchy to see if one parent is deleted by recursive + // deletion + INode tmpChild = file; + INodeDirectory tmpParent = file.getParent(); + while (true) { + if (tmpParent == null || + tmpParent.searchChildren(tmpChild.getLocalNameBytes()) < 0) { + return true; + } + if (tmpParent.isRoot()) { + break; + } + tmpChild = tmpParent; + tmpParent = tmpParent.getParent(); + } + + if (file.isWithSnapshot() && + file.getFileWithSnapshotFeature().isCurrentFileDeleted()) { return true; } return false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 5e6a4b4f78f..18b41098bd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -157,7 +157,7 @@ public class INodeDirectory extends INodeWithAdditionalFields return quota; } - private int searchChildren(byte[] name) { + int searchChildren(byte[] name) { return children == null? -1: Collections.binarySearch(children, name); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 949713e6f44..68aa85713fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -44,6 +44,9 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; @@ -1300,4 +1303,33 @@ public class DFSTestUtil { sockDir.close(); } } + + /** + * @return the node which is expected to run the recovery of the + * given block, which is known to be under construction inside the + * given NameNOde. + */ + public static DatanodeDescriptor getExpectedPrimaryNode(NameNode nn, + ExtendedBlock blk) { + BlockManager bm0 = nn.getNamesystem().getBlockManager(); + BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock()); + assertTrue("Block " + blk + " should be under construction, " + + "got: " + storedBlock, + storedBlock instanceof BlockInfoUnderConstruction); + BlockInfoUnderConstruction ucBlock = + (BlockInfoUnderConstruction)storedBlock; + // We expect that the replica with the most recent heart beat will be + // the one to be in charge of the synchronization / recovery protocol. + final DatanodeStorageInfo[] storages = ucBlock.getExpectedStorageLocations(); + DatanodeStorageInfo expectedPrimary = storages[0]; + long mostRecentLastUpdate = expectedPrimary.getDatanodeDescriptor().getLastUpdate(); + for (int i = 1; i < storages.length; i++) { + final long lastUpdate = storages[i].getDatanodeDescriptor().getLastUpdate(); + if (lastUpdate > mostRecentLastUpdate) { + expectedPrimary = storages[i]; + mostRecentLastUpdate = lastUpdate; + } + } + return expectedPrimary.getDatanodeDescriptor(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java index 45be905f89d..bd71870a5cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java @@ -50,6 +50,17 @@ public class TestCommitBlockSynchronization { FSNamesystem namesystem = new FSNamesystem(conf, image); namesystem.setImageLoaded(true); + + // set file's parent as root and put the file to inodeMap, so + // FSNamesystem's isFileDeleted() method will return false on this file + if (file.getParent() == null) { + INodeDirectory parent = mock(INodeDirectory.class); + parent.setLocalName(new byte[0]); + parent.addChild(file); + file.setParent(parent); + } + namesystem.dir.getINodeMap().put(file); + FSNamesystem namesystemSpy = spy(namesystem); BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java index d78e3a3f0a8..4cdd8092d18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.FileNotFoundException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.logging.Log; @@ -27,19 +29,30 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.StorageType; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.Node; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; @@ -49,6 +62,7 @@ import org.mockito.internal.util.reflection.Whitebox; * whole duration. */ public class TestDeleteRace { + private static final int BLOCK_SIZE = 4096; private static final Log LOG = LogFactory.getLog(TestDeleteRace.class); private static final Configuration conf = new HdfsConfiguration(); private MiniDFSCluster cluster; @@ -201,7 +215,126 @@ public class TestDeleteRace { cluster.shutdown(); } } + } + /** + * Test race between delete operation and commitBlockSynchronization method. + * See HDFS-6825. + * @param hasSnapshot + * @throws Exception + */ + private void testDeleteAndCommitBlockSynchronizationRace(boolean hasSnapshot) + throws Exception { + LOG.info("Start testing, hasSnapshot: " + hasSnapshot); + final String testPaths[] = { + "/test-file", + "/testdir/testdir1/test-file" + }; + final Path rootPath = new Path("/"); + final Configuration conf = new Configuration(); + // Disable permissions so that another user can recover the lease. + conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + FSDataOutputStream stm = null; + Map dnMap = + new HashMap(); + try { + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(3) + .build(); + cluster.waitActive(); + + DistributedFileSystem fs = cluster.getFileSystem(); + int stId = 0; + for (String testPath : testPaths) { + LOG.info("test on " + testPath + " snapshot: " + hasSnapshot); + Path fPath = new Path(testPath); + //find grandest non-root parent + Path grandestNonRootParent = fPath; + while (!grandestNonRootParent.getParent().equals(rootPath)) { + grandestNonRootParent = grandestNonRootParent.getParent(); + } + stm = fs.create(fPath); + LOG.info("test on " + testPath + " created " + fPath); + + // write a half block + AppendTestUtil.write(stm, 0, BLOCK_SIZE / 2); + stm.hflush(); + + if (hasSnapshot) { + SnapshotTestHelper.createSnapshot(fs, rootPath, + "st" + String.valueOf(stId)); + ++stId; + } + + // Look into the block manager on the active node for the block + // under construction. + NameNode nn = cluster.getNameNode(); + ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, fPath); + DatanodeDescriptor expectedPrimary = + DFSTestUtil.getExpectedPrimaryNode(nn, blk); + LOG.info("Expecting block recovery to be triggered on DN " + + expectedPrimary); + + // Find the corresponding DN daemon, and spy on its connection to the + // active. + DataNode primaryDN = cluster.getDataNode(expectedPrimary.getIpcPort()); + DatanodeProtocolClientSideTranslatorPB nnSpy = dnMap.get(primaryDN); + if (nnSpy == null) { + nnSpy = DataNodeTestUtils.spyOnBposToNN(primaryDN, nn); + dnMap.put(primaryDN, nnSpy); + } + + // Delay the commitBlockSynchronization call + DelayAnswer delayer = new DelayAnswer(LOG); + Mockito.doAnswer(delayer).when(nnSpy).commitBlockSynchronization( + Mockito.eq(blk), + Mockito.anyInt(), // new genstamp + Mockito.anyLong(), // new length + Mockito.eq(true), // close file + Mockito.eq(false), // delete block + (DatanodeID[]) Mockito.anyObject(), // new targets + (String[]) Mockito.anyObject()); // new target storages + + fs.recoverLease(fPath); + + LOG.info("Waiting for commitBlockSynchronization call from primary"); + delayer.waitForCall(); + + LOG.info("Deleting recursively " + grandestNonRootParent); + fs.delete(grandestNonRootParent, true); + + delayer.proceed(); + LOG.info("Now wait for result"); + delayer.waitForResult(); + Throwable t = delayer.getThrown(); + if (t != null) { + LOG.info("Result exception (snapshot: " + hasSnapshot + "): " + t); + } + } // end of loop each fPath + LOG.info("Now check we can restart"); + cluster.restartNameNodes(); + LOG.info("Restart finished"); + } finally { + if (stm != null) { + IOUtils.closeStream(stm); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @Test(timeout=600000) + public void testDeleteAndCommitBlockSynchonizationRaceNoSnapshot() + throws Exception { + testDeleteAndCommitBlockSynchronizationRace(false); + } + + @Test(timeout=600000) + public void testDeleteAndCommitBlockSynchronizationRaceHasSnapshot() + throws Exception { + testDeleteAndCommitBlockSynchronizationRace(true); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java index 7c87ab0d664..bba3dbb1196 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java @@ -356,7 +356,8 @@ public class TestPipelinesFailover { NameNode nn0 = cluster.getNameNode(0); ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, TEST_PATH); - DatanodeDescriptor expectedPrimary = getExpectedPrimaryNode(nn0, blk); + DatanodeDescriptor expectedPrimary = + DFSTestUtil.getExpectedPrimaryNode(nn0, blk); LOG.info("Expecting block recovery to be triggered on DN " + expectedPrimary); @@ -506,37 +507,6 @@ public class TestPipelinesFailover { } } - - - /** - * @return the node which is expected to run the recovery of the - * given block, which is known to be under construction inside the - * given NameNOde. - */ - private DatanodeDescriptor getExpectedPrimaryNode(NameNode nn, - ExtendedBlock blk) { - BlockManager bm0 = nn.getNamesystem().getBlockManager(); - BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock()); - assertTrue("Block " + blk + " should be under construction, " + - "got: " + storedBlock, - storedBlock instanceof BlockInfoUnderConstruction); - BlockInfoUnderConstruction ucBlock = - (BlockInfoUnderConstruction)storedBlock; - // We expect that the replica with the most recent heart beat will be - // the one to be in charge of the synchronization / recovery protocol. - final DatanodeStorageInfo[] storages = ucBlock.getExpectedStorageLocations(); - DatanodeStorageInfo expectedPrimary = storages[0]; - long mostRecentLastUpdate = expectedPrimary.getDatanodeDescriptor().getLastUpdate(); - for (int i = 1; i < storages.length; i++) { - final long lastUpdate = storages[i].getDatanodeDescriptor().getLastUpdate(); - if (lastUpdate > mostRecentLastUpdate) { - expectedPrimary = storages[i]; - mostRecentLastUpdate = lastUpdate; - } - } - return expectedPrimary.getDatanodeDescriptor(); - } - private DistributedFileSystem createFsAsOtherUser( final MiniDFSCluster cluster, final Configuration conf) throws IOException, InterruptedException { From 55d7fecd09a515892b2a4e8514ffdc57dfdb07f9 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Mon, 18 Aug 2014 18:09:38 +0000 Subject: [PATCH 66/67] Update CHANGES.txt for HDFS-6561 which was renamed to HADOOP-10975. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618686 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 32507184956..2a0b371e5fa 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -527,8 +527,8 @@ Release 2.6.0 - UNRELEASED HADOOP-10335. An ip whilelist based implementation to resolve Sasl properties per connection. (Benoy Antony via Arpit Agarwal) - HDFS-6561. org.apache.hadoop.util.DataChecksum should support native - checksumming (James Thomas via Colin Patrick McCabe) + HADOOP-10975. org.apache.hadoop.util.DataChecksum should support calculating + checksums in native code (James Thomas via Colin Patrick McCabe) OPTIMIZATIONS From d51f81c3b6e55feece55df5e2c052b18ca2dc768 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Mon, 18 Aug 2014 18:18:22 +0000 Subject: [PATCH 67/67] MAPREDUCE-6036. TestJobEndNotifier fails intermittently in branch-2. Contributed by chang li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1618691 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ddec273dcb1..0ff390db839 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -230,6 +230,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6024. Shortened the time when Fetcher is stuck in retrying before concluding the failure by configuration. (Yunjiong Zhao via zjshen) + MAPREDUCE-6036. TestJobEndNotifier fails intermittently in branch-2 (chang + li via jlowe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java index e143b25695d..ecfa43c447f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java @@ -270,7 +270,8 @@ public class TestJobEndNotifier extends JobEndNotifier { app.waitForInternalState(job, JobStateInternal.REBOOT); // Now shutdown. User should see FAILED state. // Unregistration fails: isLastAMRetry is recalculated, this is - app.shutDownJob(); + ///reboot will stop service internally, we don't need to shutdown twice + app.waitForServiceToStop(10000); Assert.assertFalse(app.isLastAMRetry()); // Since it's not last retry, JobEndServlet didn't called Assert.assertEquals(0, JobEndServlet.calledTimes);