diff --git a/BUILDING.txt b/BUILDING.txt index 5d331d400f3..57dad328cf4 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -5,7 +5,7 @@ Requirements: * Unix System * JDK 1.8+ -* Maven 3.0 or later +* Maven 3.3 or later * Findbugs 1.3.9 (if running findbugs) * ProtocolBuffer 2.5.0 * CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac diff --git a/LICENSE.txt b/LICENSE.txt index 52da57acb6d..969708fd1b9 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -631,9 +631,9 @@ Azure Data Lake Store - Java client SDK 2.0.11 JCodings 1.0.8 Joni 2.1.2 Mockito 1.8.5 -JUL to SLF4J bridge 1.7.10 -SLF4J API Module 1.7.10 -SLF4J LOG4J-12 Binding 1.7.10 +JUL to SLF4J bridge 1.7.25 +SLF4J API Module 1.7.25 +SLF4J LOG4J-12 Binding 1.7.25 -------------------------------------------------------------------------------- The MIT License (MIT) @@ -736,7 +736,7 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE The binary distribution of this product bundles these dependencies under the following license: -HSQLDB Database 2.0.0 +HSQLDB Database 2.3.4 -------------------------------------------------------------------------------- (HSQL License) "COPYRIGHTS AND LICENSES (based on BSD License) @@ -1711,7 +1711,6 @@ Hamcrest Core 1.3 ASM Core 5.0.4 ASM Commons 5.0.2 ASM Tree 5.0.2 -xmlenc Library 0.52 -------------------------------------------------------------------------------- (3-clause BSD) Redistribution and use in source and binary forms, with or without diff --git a/dev-support/bin/yetus-wrapper b/dev-support/bin/yetus-wrapper index ddcc7a53ab3..9f6bb331c34 100755 --- a/dev-support/bin/yetus-wrapper +++ b/dev-support/bin/yetus-wrapper @@ -14,6 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# you must be this high to ride the ride +if [[ -z "${BASH_VERSINFO[0]}" ]] \ + || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ + || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then + echo "bash v3.2+ is required. Sorry." + exit 1 +fi + set -o pipefail ## @description Print a message to stderr @@ -39,6 +47,7 @@ function yetus_abs declare obj=$1 declare dir declare fn + declare dirret if [[ ! -e ${obj} ]]; then return 1 @@ -51,7 +60,8 @@ function yetus_abs fi dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P) - if [[ $? = 0 ]]; then + dirret=$? + if [[ ${dirret} = 0 ]]; then echo "${dir}${fn}" return 0 fi @@ -63,7 +73,7 @@ WANTED="$1" shift ARGV=("$@") -HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.3.0} +HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.4.0} BIN=$(yetus_abs "${BASH_SOURCE-$0}") BINDIR=$(dirname "${BIN}") @@ -85,7 +95,8 @@ if [[ ! -d "${HADOOP_PATCHPROCESS}" ]]; then fi mytmpdir=$(yetus_abs "${HADOOP_PATCHPROCESS}") -if [[ $? != 0 ]]; then +ret=$? +if [[ ${ret} != 0 ]]; then yetus_error "yetus-dl: Unable to cwd to ${HADOOP_PATCHPROCESS}" exit 1 fi @@ -108,15 +119,13 @@ TARBALL="yetus-${HADOOP_YETUS_VERSION}-bin.tar" GPGBIN=$(command -v gpg) CURLBIN=$(command -v curl) -pushd "${HADOOP_PATCHPROCESS}" >/dev/null -if [[ $? != 0 ]]; then +if ! pushd "${HADOOP_PATCHPROCESS}" >/dev/null; then yetus_error "ERROR: yetus-dl: Cannot pushd to ${HADOOP_PATCHPROCESS}" exit 1 fi if [[ -n "${CURLBIN}" ]]; then - "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz" - if [[ $? != 0 ]]; then + if ! "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz"; then yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz" exit 1 fi @@ -126,40 +135,33 @@ else fi if [[ -n "${GPGBIN}" ]]; then - mkdir -p .gpg - if [[ $? != 0 ]]; then + if ! mkdir -p .gpg; then yetus_error "ERROR: yetus-dl: Unable to create ${HADOOP_PATCHPROCESS}/.gpg" exit 1 fi - chmod -R 700 .gpg - if [[ $? != 0 ]]; then + if ! chmod -R 700 .gpg; then yetus_error "ERROR: yetus-dl: Unable to chmod ${HADOOP_PATCHPROCESS}/.gpg" exit 1 fi - "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS - if [[ $? != 0 ]]; then + if ! "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS; then yetus_error "ERROR: yetus-dl: unable to fetch https://dist.apache.org/repos/dist/release/yetus/KEYS" exit 1 fi - "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc" - if [[ $? != 0 ]]; then + if ! "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc"; then yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc" exit 1 fi - "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --import "${HADOOP_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1 - if [[ $? != 0 ]]; then + if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --import "${HADOOP_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1; then yetus_error "ERROR: yetus-dl: gpg unable to import ${HADOOP_PATCHPROCESS}/KEYS_YETUS" exit 1 fi - "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1 - if [[ $? != 0 ]]; then + if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1; then yetus_error "ERROR: yetus-dl: gpg verify of tarball in ${HADOOP_PATCHPROCESS} failed" exit 1 fi fi -gunzip -c "${TARBALL}.gz" | tar xpf - -if [[ $? != 0 ]]; then +if ! (gunzip -c "${TARBALL}.gz" | tar xpf -); then yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HADOOP_PATCHPROCESS} to try again." exit 1 fi diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index e3e781042d9..a135c61b970 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -81,13 +81,17 @@ RUN apt-get -q install --no-install-recommends -y oracle-java8-installer #### # Apps that require Java ### -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ - ant \ - maven +RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant -# Fixing the Apache commons / Maven dependency problem under Ubuntu: -# See http://wiki.apache.org/commons/VfsProblems -RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . +###### +# Install Apache Maven +###### +RUN mkdir -p /opt/maven && \ + curl -L -s -S \ + http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \ + -o /opt/maven.tar.gz && \ + tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven +ENV MAVEN_HOME /opt/maven ###### # Install findbugs diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml index 02e5824d00a..99fd0c2bfa8 100644 --- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml +++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml @@ -145,6 +145,11 @@ hadoop-common test + + org.apache.hadoop + hadoop-hdfs-client + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index c58ac385831..b6197b332bf 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -218,10 +218,6 @@ javax.servlet javax.servlet-api - - xmlenc - xmlenc - + fs.defaultFS @@ -956,13 +955,8 @@ configuration of AWS access key ID and secret access key in environment variables named AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider: - a shared instance of - com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS - SDK, which supports use of instance profile credentials if running - in an EC2 VM. Using this shared instance potentially reduces load - on the EC2 instance metadata service for multi-threaded - applications. + 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use + of instance profile credentials if running in an EC2 VM. @@ -2202,6 +2196,8 @@ The KeyProvider to use when managing zone keys, and interacting with encryption keys when reading and writing to an encryption zone. + For hdfs clients, the provider path will be same as namenode's + provider path. @@ -2454,6 +2450,7 @@ + fs.adl.impl org.apache.hadoop.fs.adl.AdlFileSystem @@ -2463,6 +2460,68 @@ fs.AbstractFileSystem.adl.impl org.apache.hadoop.fs.adl.Adl + + + adl.feature.ownerandgroup.enableupn + false + + When true : User and Group in FileStatus/AclStatus response is + represented as user friendly name as per Azure AD profile. + + When false (default) : User and Group in FileStatus/AclStatus + response is represented by the unique identifier from Azure AD + profile (Object ID as GUID). + + For optimal performance, false is recommended. + + + + + fs.adl.oauth2.access.token.provider.type + ClientCredential + + Defines Azure Active Directory OAuth2 access token provider type. + Supported types are ClientCredential, RefreshToken, and Custom. + The ClientCredential type requires property fs.adl.oauth2.client.id, + fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url. + The RefreshToken type requires property fs.adl.oauth2.client.id and + fs.adl.oauth2.refresh.token. + The Custom type requires property fs.adl.oauth2.access.token.provider. + + + + + fs.adl.oauth2.client.id + + The OAuth2 client id. + + + + fs.adl.oauth2.credential + + The OAuth2 access key. + + + + fs.adl.oauth2.refresh.url + + The OAuth2 token endpoint. + + + + fs.adl.oauth2.refresh.token + + The OAuth2 refresh token. + + + + fs.adl.oauth2.access.token.provider + + + The class name of the OAuth2 access token provider. + + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md index 24de93e924a..e1aad5ac8c7 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md @@ -168,9 +168,9 @@ Some products such as Apache Oozie which access the services of Hadoop on behalf Because the DataNode data transfer protocol does not use the Hadoop RPC framework, DataNodes must authenticate themselves using privileged ports which are specified by `dfs.datanode.address` and `dfs.datanode.http.address`. This authentication is based on the assumption that the attacker won't be able to get root privileges on DataNode hosts. -When you execute the `hdfs datanode` command as root, the server process binds privileged ports at first, then drops privilege and runs as the user account specified by `HADOOP_SECURE_DN_USER`. This startup process uses [the jsvc program](https://commons.apache.org/proper/commons-daemon/jsvc.html "Link to Apache Commons Jsvc") installed to `JSVC_HOME`. You must specify `HADOOP_SECURE_DN_USER` and `JSVC_HOME` as environment variables on start up (in `hadoop-env.sh`). +When you execute the `hdfs datanode` command as root, the server process binds privileged ports at first, then drops privilege and runs as the user account specified by `HDFS_DATANODE_SECURE_USER`. This startup process uses [the jsvc program](https://commons.apache.org/proper/commons-daemon/jsvc.html "Link to Apache Commons Jsvc") installed to `JSVC_HOME`. You must specify `HDFS_DATANODE_SECURE_USER` and `JSVC_HOME` as environment variables on start up (in `hadoop-env.sh`). -As of version 2.6.0, SASL can be used to authenticate the data transfer protocol. In this configuration, it is no longer required for secured clusters to start the DataNode as root using `jsvc` and bind to privileged ports. To enable SASL on data transfer protocol, set `dfs.data.transfer.protection` in hdfs-site.xml, set a non-privileged port for `dfs.datanode.address`, set `dfs.http.policy` to `HTTPS_ONLY` and make sure the `HADOOP_SECURE_DN_USER` environment variable is not defined. Note that it is not possible to use SASL on data transfer protocol if `dfs.datanode.address` is set to a privileged port. This is required for backwards-compatibility reasons. +As of version 2.6.0, SASL can be used to authenticate the data transfer protocol. In this configuration, it is no longer required for secured clusters to start the DataNode as root using `jsvc` and bind to privileged ports. To enable SASL on data transfer protocol, set `dfs.data.transfer.protection` in hdfs-site.xml, set a non-privileged port for `dfs.datanode.address`, set `dfs.http.policy` to `HTTPS_ONLY` and make sure the `HDFS_DATANODE_SECURE_USER` environment variable is not defined. Note that it is not possible to use SASL on data transfer protocol if `dfs.datanode.address` is set to a privileged port. This is required for backwards-compatibility reasons. In order to migrate an existing cluster that used root authentication to start using SASL instead, first ensure that version 2.6.0 or later has been deployed to all cluster nodes as well as any external applications that need to connect to the cluster. Only versions 2.6.0 and later of the HDFS client can connect to a DataNode that uses SASL for authentication of data transfer protocol, so it is vital that all callers have the correct version before migrating. After version 2.6.0 or later has been deployed everywhere, update configuration of any external applications to enable SASL. If an HDFS client is enabled for SASL, then it can connect successfully to a DataNode running with either root authentication or SASL authentication. Changing configuration for all clients guarantees that subsequent configuration changes on DataNodes will not disrupt the applications. Finally, each individual DataNode can be migrated by changing its configuration and restarting. It is acceptable to have a mix of some DataNodes running with root authentication and some DataNodes running with SASL authentication temporarily during this migration period, because an HDFS client enabled for SASL can connect to both. @@ -293,7 +293,7 @@ The following settings allow configuring SSL access to the NameNode web UI (opti | `dfs.encrypt.data.transfer.algorithm` | | optionally set to `3des` or `rc4` when using data encryption to control encryption algorithm | | `dfs.encrypt.data.transfer.cipher.suites` | | optionally set to `AES/CTR/NoPadding` to activate AES encryption when using data encryption | | `dfs.encrypt.data.transfer.cipher.key.bitlength` | | optionally set to `128`, `192` or `256` to control key bit length when using AES with data encryption | -| `dfs.data.transfer.protection` | | `authentication` : authentication only; `integrity` : integrity check in addition to authentication; `privacy` : data encryption in addition to integrity This property is unspecified by default. Setting this property enables SASL for authentication of data transfer protocol. If this is enabled, then `dfs.datanode.address` must use a non-privileged port, `dfs.http.policy` must be set to `HTTPS_ONLY` and the `HADOOP_SECURE_DN_USER` environment variable must be undefined when starting the DataNode process. | +| `dfs.data.transfer.protection` | | `authentication` : authentication only; `integrity` : integrity check in addition to authentication; `privacy` : data encryption in addition to integrity This property is unspecified by default. Setting this property enables SASL for authentication of data transfer protocol. If this is enabled, then `dfs.datanode.address` must use a non-privileged port, `dfs.http.policy` must be set to `HTTPS_ONLY` and the `HDFS_DATANODE_SECURE_USER` environment variable must be undefined when starting the DataNode process. | ### WebHDFS @@ -413,7 +413,7 @@ Set the environment variable `HADOOP_JAAS_DEBUG` to `true`. export HADOOP_JAAS_DEBUG=true ``` -Edit the `log4j.properties` file to log Hadoop's security package at `DEBUG` level. +Edit the `log4j.properties` file to log Hadoop's security package at `DEBUG` level. ``` log4j.logger.org.apache.hadoop.security=DEBUG @@ -434,19 +434,19 @@ It contains a series of probes for the JVM's configuration and the environment, dumps out some system files (`/etc/krb5.conf`, `/etc/ntp.conf`), prints out some system state and then attempts to log in to Kerberos as the current user, or a specific principal in a named keytab. - + The output of the command can be used for local diagnostics, or forwarded to whoever supports the cluster. The `KDiag` command has its own entry point; it is currently not hooked up -to the end-user CLI. +to the end-user CLI. It is invoked simply by passing its full classname to one of the `bin/hadoop`, `bin/hdfs` or `bin/yarn` commands. Accordingly, it will display the kerberos client state of the command used to invoke it. ``` -hadoop org.apache.hadoop.security.KDiag +hadoop org.apache.hadoop.security.KDiag hdfs org.apache.hadoop.security.KDiag yarn org.apache.hadoop.security.KDiag ``` @@ -557,8 +557,8 @@ hdfs org.apache.hadoop.security.KDiag --resource hbase-default.xml --resource hb yarn org.apache.hadoop.security.KDiag --resource yarn-default.xml --resource yarn-site.xml ``` -For extra logging during the operation, set the logging and `HADOOP_JAAS_DEBUG` -environment variable to the values listed in "Troubleshooting". The JVM +For extra logging during the operation, set the logging and `HADOOP_JAAS_DEBUG` +environment variable to the values listed in "Troubleshooting". The JVM options are automatically set in KDiag. #### `--secure`: Fail if the command is not executed on a secure cluster. @@ -589,7 +589,7 @@ hdfs org.apache.hadoop.security.KDiag \ --keylen 1024 \ --keytab zk.service.keytab --principal zookeeper/devix.example.org@REALM ``` - + This attempts to to perform all diagnostics without failing early, load in the HDFS and YARN XML resources, require a minimum key length of 1024 bytes, and log in as the principal `zookeeper/devix.example.org@REALM`, whose key must be in diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md index 098f2742ec3..97f9e9aa92a 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md @@ -32,6 +32,8 @@ HADOOP_CLIENT_OPTS="-Xmx1g -Dhadoop.socks.server=localhost:4000" hadoop fs -ls / will increase the memory and send this command via a SOCKS proxy server. +NOTE: If 'YARN_CLIENT_OPTS' is defined, it will replace 'HADOOP_CLIENT_OPTS' when commands are run with 'yarn'. + ### `(command)_(subcommand)_OPTS` It is also possible to set options on a per subcommand basis. This allows for one to create special options for particular cases. The first part of the pattern is the command being used, but all uppercase. The second part of the command is the subcommand being used. Then finally followed by the string `_OPT`. @@ -103,13 +105,15 @@ In addition, daemons that run in an extra security mode also support `(command)_ Apache Hadoop provides a way to do a user check per-subcommand. While this method is easily circumvented and should not be considered a security-feature, it does provide a mechanism by which to prevent accidents. For example, setting `HDFS_NAMENODE_USER=hdfs` will make the `hdfs namenode` and `hdfs --daemon start namenode` commands verify that the user running the commands are the hdfs user by checking the `USER` environment variable. This also works for non-daemons. Setting `HADOOP_DISTCP_USER=jane` will verify that `USER` is set to `jane` before being allowed to execute the `hadoop distcp` command. -If a \_USER environment variable exists and commands are run with a privilege (e.g., as root; see hadoop_privilege_check in the API documentation), execution will switch to the specified user. For commands that support user account switching for security and therefore have a SECURE\_USER variable, the base \_USER variable needs to be the user that is expected to be used to switch to the SECURE\_USER account. For example: +If a \_USER environment variable exists and commands are run with a privilege (e.g., as root; see hadoop_privilege_check in the API documentation), execution will switch to the specified user first. For commands that support user account switching for security reasons and therefore have a SECURE\_USER variable (see more below), the base \_USER variable needs to be the user that is expected to be used to switch to the SECURE\_USER account. For example: ```bash HDFS_DATANODE_USER=root HDFS_DATANODE_SECURE_USER=hdfs ``` +will force 'hdfs --daemon start datanode' to be root, but will eventually switch to the hdfs user after the privileged work has been completed. + Be aware that if the \-\-workers flag is used, the user switch happens *after* ssh is invoked. The multi-daemon start and stop commands in sbin will, however, switch (if appropriate) prior and will therefore use the keys of the specified \_USER. ## Developer and Advanced Administrator Environment @@ -172,7 +176,7 @@ which will result in the output of: world I see you ``` -It is also possible to add the new subcommands to the usage output. The `hadoop_add_subcommand` function adds text to the usage output. Utilizing the standard HADOOP_SHELL_EXECNAME variable, we can limit which command gets our new function. +It is also possible to add the new subcommands to the usage output. The `hadoop_add_subcommand` function adds text to the usage output. Utilizing the standard HADOOP\_SHELL\_EXECNAME variable, we can limit which command gets our new function. ```bash if [[ "${HADOOP_SHELL_EXECNAME}" = "yarn" ]]; then @@ -191,12 +195,16 @@ function hdfs_subcommand_fetchdt ... will replace the existing `hdfs fetchdt` subcommand with a custom one. -Some key environment variables related to Dynamic Subcommands: +Some key environment variables for Dynamic Subcommands: * HADOOP\_CLASSNAME This is the name of the Java class to use when program execution continues. +* HADOOP\_PRIV\_CLASSNAME + +This is the name of the Java class to use when a daemon is expected to be run in a privileged mode. (See more below.) + * HADOOP\_SHELL\_EXECNAME This is the name of the script that is being executed. It will be one of hadoop, hdfs, mapred, or yarn. @@ -209,13 +217,13 @@ This is the subcommand that was passed on the command line. This array contains the argument list after the Apache Hadoop common argument processing has taken place and is the same list that is passed to the subcommand function as arguments. For example, if `hadoop --debug subcmd 1 2 3` has been executed on the command line, then `${HADOOP_SUBCMD_ARGS[0]}` will be 1 and `hadoop_subcommand_subcmd` will also have $1 equal to 1. This array list MAY be modified by subcommand functions to add or delete values from the argument list for further processing. +* HADOOP\_SECURE\_CLASSNAME + +If this subcommand runs a service that supports the secure mode, this variable should be set to the classname of the secure version. + * HADOOP\_SUBCMD\_SECURESERVICE -If this command should/will be executed as a secure daemon, set this to true. - -* HADOOP\_SUBCMD\_SECUREUSER - -If this command should/will be executed as a secure daemon, set the user name to be used. +Setting this to true will force the subcommand to run in secure mode regardless of hadoop\_detect\_priv\_subcmd. It is expected that HADOOP\_SECURE\_USER will be set to the user that will be executing the final process. See more about secure mode. * HADOOP\_SUBCMD\_SUPPORTDAEMONIZATION @@ -226,3 +234,12 @@ If this command can be executed as a daemon, set this to true. This is the full content of the command line, prior to any parsing done. It will contain flags such as `--debug`. It MAY NOT be manipulated. The Apache Hadoop runtime facilities require functions exit if no further processing is required. For example, in the hello example above, Java and other facilities were not required so a simple `exit $?` was sufficient. However, if the function were to utilize `HADOOP_CLASSNAME`, then program execution must continue so that Java with the Apache Hadoop-specific parameters will be launched against the given Java class. Another example would be in the case of an unrecoverable error. It is the function's responsibility to print an appropriate message (preferably using the hadoop_error API call) and exit appropriately. + +### Running with Privilege (Secure Mode) + +Some daemons, such as the DataNode and the NFS gateway, may be run in a privileged mode. This means that they are expected to be launched as root and (by default) switched to another userid via jsvc. This allows for these daemons to grab a low, privileged port and then drop superuser privileges during normal execution. Running with privilege is also possible for 3rd parties utilizing Dynamic Subcommands. If the following are true: + + * (command)\_(subcommand)\_SECURE\_USER environment variable is defined and points to a valid username + * HADOOP\_SECURE\_CLASSNAME is defined and points to a valid Java class + +then the shell scripts will attempt to run the class as a command with privilege as it would the built-ins. In general, users are expected to define the \_SECURE\_USER variable and developers define the \_CLASSNAME in their shell script bootstrap. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index cbfb6d1a7c2..8524973c2c0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -105,7 +105,7 @@ public void initializeMemberVariables() { // ADL properties are in a different subtree // - org.apache.hadoop.hdfs.web.ADLConfKeys xmlPrefixToSkipCompare.add("adl."); - xmlPropsToSkipCompare.add("fs.adl.impl"); + xmlPrefixToSkipCompare.add("fs.adl."); xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl"); // Azure properties are in a different class diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java index 427b0e972d2..71d27067bd5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java @@ -126,4 +126,46 @@ public void testMkdirSlashHandling() throws Throwable { assertPathExists("check path existence without trailing slash failed", path("testmkdir/b")); } + + @Test + public void testMkdirsPopulatingAllNonexistentAncestors() throws IOException { + describe("Verify mkdir will populate all its non-existent ancestors"); + final FileSystem fs = getFileSystem(); + + final Path parent = path("testMkdirsPopulatingAllNonexistentAncestors"); + assertTrue(fs.mkdirs(parent)); + assertPathExists(parent + " should exist before making nested dir", parent); + + Path nested = path(parent + "/a/b/c/d/e/f/g/h/i/j/k/L"); + assertTrue(fs.mkdirs(nested)); + while (nested != null && !nested.equals(parent) && !nested.isRoot()) { + assertPathExists(nested + " nested dir should exist", nested); + nested = nested.getParent(); + } + } + + @Test + public void testMkdirsDoesNotRemoveParentDirectories() throws IOException { + describe("Verify mkdir will make its parent existent"); + final FileSystem fs = getFileSystem(); + + final Path parent = path("testMkdirsDoesNotRemoveParentDirectories"); + assertTrue(fs.mkdirs(parent)); + + Path p = parent; + for (int i = 0; i < 10; i++) { + assertTrue(fs.mkdirs(p)); + assertPathExists(p + " should exist after mkdir(" + p + ")", p); + p = path(p + "/dir-" + i); + } + + // After mkdirs(sub-directory), its parent directory still exists + p = p.getParent(); + while (p != null && !p.equals(parent) && !p.isRoot()) { + assertPathExists("Path " + p + " should exist", p); + assertIsDirectory(p); + p = p.getParent(); + } + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index 7f7fcf32ae4..db31993a7e3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -18,12 +18,16 @@ package org.apache.hadoop.io.erasurecode; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; +import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; @@ -70,20 +74,73 @@ public void testDedicatedRawCoderKey() { numDataUnit, numParityUnit); String dummyFactName = "DummyNoneExistingFactory"; - // set the dummy factory to rs-legacy and create a raw coder - // with rs, which is OK as the raw coder key is not used + // set the dummy factory to raw coders then fail to create any rs raw coder. conf.set(CodecUtil. - IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName); - RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf, - ErasureCodeConstants.RS_CODEC_NAME, coderOptions); - Assert.assertTrue(encoder instanceof RSRawEncoder); + IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, dummyFactName); + try { + CodecUtil.createRawEncoder(conf, + ErasureCodeConstants.RS_CODEC_NAME, coderOptions); + Assert.fail(); + } catch (Exception e) { + GenericTestUtils.assertExceptionContains( + "Fail to create raw erasure encoder with given codec: rs", e); + } + // now create the raw coder with rs-legacy, which should throw exception + conf.set(CodecUtil. + IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY, dummyFactName); try { CodecUtil.createRawEncoder(conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions); Assert.fail(); } catch (Exception e) { - GenericTestUtils.assertExceptionContains("Failed to create raw coder", e); + GenericTestUtils.assertExceptionContains( + "Fail to create raw erasure encoder with given codec: rs", e); } } + + @Test + public void testFallbackCoders() { + ErasureCoderOptions coderOptions = new ErasureCoderOptions( + numDataUnit, numParityUnit); + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, + RSRawErasureCoderFactory.class.getCanonicalName() + + "," + NativeRSRawErasureCoderFactory.class.getCanonicalName()); + // should return default raw coder of rs codec + RawErasureEncoder encoder = CodecUtil.createRawEncoder( + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); + Assert.assertTrue(encoder instanceof RSRawEncoder); + RawErasureDecoder decoder = CodecUtil.createRawDecoder( + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); + Assert.assertTrue(decoder instanceof RSRawDecoder); + } + + @Test + public void testLegacyCodecFallback() { + ErasureCoderOptions coderOptions = new ErasureCoderOptions( + numDataUnit, numParityUnit); + // should return default raw coder of rs-legacy codec + RawErasureEncoder encoder = CodecUtil.createRawEncoder( + conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions); + Assert.assertTrue(encoder instanceof RSRawEncoderLegacy); + RawErasureDecoder decoder = CodecUtil.createRawDecoder( + conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions); + Assert.assertTrue(decoder instanceof RSRawDecoderLegacy); + } + + @Test + public void testIgnoreInvalidCodec() { + ErasureCoderOptions coderOptions = new ErasureCoderOptions( + numDataUnit, numParityUnit); + conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY, + "invalid-codec," + + "org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory"); + // should return second coder specified by IO_ERASURECODE_CODEC_CODERS + RawErasureEncoder encoder = CodecUtil.createRawEncoder( + conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + Assert.assertTrue(encoder instanceof XORRawEncoder); + RawErasureDecoder decoder = CodecUtil.createRawDecoder( + conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + Assert.assertTrue(decoder instanceof XORRawDecoder); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java index 91c4ba8c1a1..a475a39f678 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java @@ -50,7 +50,7 @@ public void testCodingDirectBufferWithConf_10x4_erasing_d0() { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index 3b18347d769..11abac40b82 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -57,7 +57,7 @@ public void testCodingDirectBufferWithConf_10x4_erasing_d0() { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java index 5b75e33e318..00c216590a8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java @@ -39,6 +39,7 @@ static class MyMetrics { @Metric({"Counter2", "Counter2 desc"}) MutableCounterLong c2; @Metric MutableGaugeInt g1, g2; @Metric("g3 desc") MutableGaugeLong g3; + @Metric("g4 desc") MutableGaugeFloat g4; @Metric MutableRate r1; @Metric MutableStat s1; @Metric MutableRates rs1; @@ -53,6 +54,7 @@ static class MyMetrics { metrics.g1.incr(); metrics.g2.incr(); metrics.g3.incr(); + metrics.g4.incr(); metrics.r1.add(1); metrics.s1.add(1); metrics.rs1.add("rs1", 1); @@ -64,6 +66,7 @@ static class MyMetrics { verify(rb).addGauge(info("G1", "G1"), 1); verify(rb).addGauge(info("G2", "G2"), 1); verify(rb).addGauge(info("G3", "g3 desc"), 1L); + verify(rb).addGauge(info("G4", "g4 desc"), 1f); verify(rb).addCounter(info("R1NumOps", "Number of ops for r1"), 1L); verify(rb).addGauge(info("R1AvgTime", "Average time for r1"), 1.0); verify(rb).addCounter(info("S1NumOps", "Number of ops for s1"), 1L); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java index d91692807eb..73ea43f69ad 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java @@ -42,13 +42,15 @@ public class TestMetricsRegistry { r.newCounter("c2", "c2 desc", 2L); r.newGauge("g1", "g1 desc", 3); r.newGauge("g2", "g2 desc", 4L); + r.newGauge("g3", "g3 desc", 5f); r.newStat("s1", "s1 desc", "ops", "time"); - assertEquals("num metrics in registry", 5, r.metrics().size()); + assertEquals("num metrics in registry", 6, r.metrics().size()); assertTrue("c1 found", r.get("c1") instanceof MutableCounterInt); assertTrue("c2 found", r.get("c2") instanceof MutableCounterLong); assertTrue("g1 found", r.get("g1") instanceof MutableGaugeInt); assertTrue("g2 found", r.get("g2") instanceof MutableGaugeLong); + assertTrue("g3 found", r.get("g3") instanceof MutableGaugeFloat); assertTrue("s1 found", r.get("s1") instanceof MutableStat); expectMetricsException("Metric name c1 already exists", new Runnable() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java index 1faa361defc..fd716ae7116 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java @@ -59,6 +59,7 @@ public class TestMutableMetrics { registry.newCounter("c2", "long counter", 2L); registry.newGauge("g1", "int gauge", 3); registry.newGauge("g2", "long gauge", 4L); + registry.newGauge("g3", "float gauge", 5f); registry.newStat("s1", "stat", "Ops", "Time", true).add(0); registry.newRate("s2", "stat", false).add(0); @@ -74,6 +75,7 @@ public class TestMutableMetrics { verify(mb).addCounter(info("c2", "long counter"), 2L); verify(mb).addGauge(info("g1", "int gauge"), 3); verify(mb).addGauge(info("g2", "long gauge"), 4L); + verify(mb).addGauge(info("g3", "float gauge"), 5f); verify(mb).addCounter(info("S1NumOps", "Number of ops for stat"), 1L); verify(mb).addGauge(eq(info("S1AvgTime", "Average time for stat")), eq(0.0, EPSILON)); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/FakeTimer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/FakeTimer.java index 66386fd2b2c..1b17ce7cc9d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/FakeTimer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/FakeTimer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.util; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -28,25 +29,39 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class FakeTimer extends Timer { - private long nowMillis; + private long nowNanos; /** Constructs a FakeTimer with a non-zero value */ public FakeTimer() { - nowMillis = 1000; // Initialize with a non-trivial value. + // Initialize with a non-trivial value. + nowNanos = TimeUnit.MILLISECONDS.toNanos(1000); } @Override public long now() { - return nowMillis; + return TimeUnit.NANOSECONDS.toMillis(nowNanos); } @Override public long monotonicNow() { - return nowMillis; + return TimeUnit.NANOSECONDS.toMillis(nowNanos); + } + + @Override + public long monotonicNowNanos() { + return nowNanos; } /** Increases the time by milliseconds */ public void advance(long advMillis) { - nowMillis += advMillis; + nowNanos += TimeUnit.MILLISECONDS.toNanos(advMillis); + } + + /** + * Increases the time by nanoseconds. + * @param advNanos Nanoseconds to advance by. + */ + public void advanceNanos(long advNanos) { + nowNanos += advNanos; } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLightWeightCache.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLightWeightCache.java index dff6937317a..de6181810db 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLightWeightCache.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLightWeightCache.java @@ -213,7 +213,7 @@ private static class LightWeightCacheTestCase implements GSet(tablelength, sizeLimit, - creationExpirationPeriod, 0, new LightWeightCache.Clock() { - @Override - long currentTime() { - return currentTestTime; - } - }); + creationExpirationPeriod, 0, fakeTimer); Assert.assertEquals(0, cache.size()); } @@ -247,7 +242,7 @@ private boolean containsTest(IntEntry key) { } else { final IntEntry h = hashMap.remove(key); if (h != null) { - Assert.assertTrue(cache.isExpired(h, currentTestTime)); + Assert.assertTrue(cache.isExpired(h, fakeTimer.monotonicNowNanos())); } } return c; @@ -266,7 +261,7 @@ private IntEntry getTest(IntEntry key) { } else { final IntEntry h = hashMap.remove(key); if (h != null) { - Assert.assertTrue(cache.isExpired(h, currentTestTime)); + Assert.assertTrue(cache.isExpired(h, fakeTimer.monotonicNowNanos())); } } return c; @@ -286,7 +281,7 @@ private IntEntry putTest(IntEntry entry) { final IntEntry h = hashMap.put(entry); if (h != null && h != entry) { // if h == entry, its expiration time is already updated - Assert.assertTrue(cache.isExpired(h, currentTestTime)); + Assert.assertTrue(cache.isExpired(h, fakeTimer.monotonicNowNanos())); } } return c; @@ -305,7 +300,7 @@ private IntEntry removeTest(IntEntry key) { } else { final IntEntry h = hashMap.remove(key); if (h != null) { - Assert.assertTrue(cache.isExpired(h, currentTestTime)); + Assert.assertTrue(cache.isExpired(h, fakeTimer.monotonicNowNanos())); } } return c; @@ -339,7 +334,7 @@ boolean tossCoin() { } void check() { - currentTestTime += ran.nextInt() & 0x3; + fakeTimer.advanceNanos(ran.nextInt() & 0x3); //test size sizeTest(); diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_get_verify_uservar.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_build_custom_subcmd_var.bats similarity index 89% rename from hadoop-common-project/hadoop-common/src/test/scripts/hadoop_get_verify_uservar.bats rename to hadoop-common-project/hadoop-common/src/test/scripts/hadoop_build_custom_subcmd_var.bats index 091fd30f96f..76fc1bffae5 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_get_verify_uservar.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_build_custom_subcmd_var.bats @@ -15,7 +15,7 @@ load hadoop-functions_test_helper -@test "hadoop_get_verify_uservar" { - run hadoop_get_verify_uservar cool program +@test "hadoop_build_custom_subcmd_var" { + run hadoop_build_custom_subcmd_var cool program USER [ "${output}" = "COOL_PROGRAM_USER" ] } diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_detect_priv_subcmd.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_detect_priv_subcmd.bats new file mode 100755 index 00000000000..6b5c0eca7a1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_detect_priv_subcmd.bats @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load hadoop-functions_test_helper + +@test "hadoop_detect_priv_subcmd (no classname) " { + run hadoop_detect_priv_subcmd test app + [ "${status}" = "1" ] +} + +@test "hadoop_detect_priv_subcmd (classname; no user) " { + export HADOOP_SECURE_CLASSNAME=fake + run hadoop_detect_priv_subcmd test app + [ "${status}" = "1" ] +} + +@test "hadoop_detect_priv_subcmd (classname; user) " { + export HADOOP_SECURE_CLASSNAME=fake + export TEST_APP_SECURE_USER=test + run hadoop_detect_priv_subcmd test app + [ "${status}" = "0" ] +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user_perm.bats similarity index 68% rename from hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user.bats rename to hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user_perm.bats index ac9fa9f6595..88a6503cf2d 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user_perm.bats @@ -15,39 +15,39 @@ load hadoop-functions_test_helper -@test "hadoop_verify_user (hadoop: no setting)" { - run hadoop_verify_user hadoop test +@test "hadoop_verify_user_perm (hadoop: no setting)" { + run hadoop_verify_user_perm hadoop test [ "${status}" = "0" ] } -@test "hadoop_verify_user (yarn: no setting)" { - run hadoop_verify_user yarn test +@test "hadoop_verify_user_perm (yarn: no setting)" { + run hadoop_verify_user_perm yarn test [ "${status}" = "0" ] } -@test "hadoop_verify_user (hadoop: allow)" { +@test "hadoop_verify_user_perm (hadoop: allow)" { HADOOP_TEST_USER=${USER} - run hadoop_verify_user hadoop test + run hadoop_verify_user_perm hadoop test [ "${status}" = "0" ] } -@test "hadoop_verify_user (yarn: allow)" { +@test "hadoop_verify_user_perm (yarn: allow)" { YARN_TEST_USER=${USER} - run hadoop_verify_user yarn test + run hadoop_verify_user_perm yarn test [ "${status}" = "0" ] } # colon isn't a valid username, so let's use it # this should fail regardless of who the user is # that is running the test code -@test "hadoop_verify_user (hadoop: disallow)" { +@test "hadoop_verify_user_perm (hadoop: disallow)" { HADOOP_TEST_USER=: - run hadoop_verify_user hadoop test + run hadoop_verify_user_perm hadoop test [ "${status}" = "1" ] } -@test "hadoop_verify_user (yarn: disallow)" { +@test "hadoop_verify_user_perm (yarn: disallow)" { YARN_TEST_USER=: - run hadoop_verify_user yarn test + run hadoop_verify_user_perm yarn test [ "${status}" = "1" ] } diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user_resolves.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user_resolves.bats new file mode 100644 index 00000000000..f1263df26f4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user_resolves.bats @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load hadoop-functions_test_helper + +@test "hadoop_verify_user_resolves (bad: null)" { + run hadoop_verify_user_resolves + [ "${status}" = "1" ] +} + +@test "hadoop_verify_user_resolves (bad: var string)" { + run hadoop_verify_user_resolves PrObAbLyWiLlNoTeXiSt + [ "${status}" = "1" ] +} + +@test "hadoop_verify_user_resolves (bad: number as var)" { + run hadoop_verify_user_resolves 501 + [ "${status}" = "1" ] +} + +@test "hadoop_verify_user_resolves (good: name)" { + myvar=$(id -u -n) + run hadoop_verify_user_resolves myvar + [ "${status}" = "0" ] +} + +@test "hadoop_verify_user_resolves (skip: number)" { + skip "id on uids is not platform consistent" + myvar=1 + run hadoop_verify_user_resolves myvar + [ "${status}" = "0" ] +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index aaf8bdd6ba4..ef499505ece 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -67,6 +67,7 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; @@ -160,6 +161,7 @@ import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; @@ -197,6 +199,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, public static final Logger LOG = LoggerFactory.getLogger(DFSClient.class); // 1 hour public static final long SERVER_DEFAULTS_VALIDITY_PERIOD = 60 * 60 * 1000L; + private static final String DFS_KMS_PREFIX = "dfs-kms-"; private final Configuration conf; private final Tracer tracer; @@ -214,7 +217,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, final SocketFactory socketFactory; final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure; private final FileSystem.Statistics stats; - private final String authority; + private final URI namenodeUri; private final Random r = new Random(); private SocketAddress[] localInterfaceAddrs; private DataEncryptionKey encryptionKey; @@ -228,6 +231,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL; private static volatile ThreadPoolExecutor STRIPED_READ_THREAD_POOL; private final int smallBufferSize; + private URI keyProviderUri = null; public DfsClientConf getConf() { return dfsClientConf; @@ -298,7 +302,7 @@ public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode, this.ugi = UserGroupInformation.getCurrentUser(); - this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority(); + this.namenodeUri = nameNodeUri; this.clientName = "DFSClient_" + dfsClientConf.getTaskId() + "_" + ThreadLocalRandom.current().nextInt() + "_" + Thread.currentThread().getId(); @@ -454,7 +458,8 @@ void checkOpen() throws IOException { * be returned until all output streams are closed. */ public LeaseRenewer getLeaseRenewer() { - return LeaseRenewer.getInstance(authority, ugi, this); + return LeaseRenewer.getInstance( + namenodeUri != null ? namenodeUri.getAuthority() : "null", ugi, this); } /** Get a lease and start automatic renewal */ @@ -1185,13 +1190,31 @@ public DFSOutputStream create(String src, FsPermission permission, long blockSize, Progressable progress, int buffersize, ChecksumOpt checksumOpt, InetSocketAddress[] favoredNodes) throws IOException { + return create(src, permission, flag, createParent, replication, blockSize, + progress, buffersize, checksumOpt, favoredNodes, null); + } + + + /** + * Same as {@link #create(String, FsPermission, EnumSet, boolean, short, long, + * Progressable, int, ChecksumOpt, InetSocketAddress[])} with the addition of + * ecPolicyName that is used to specify a specific erasure coding policy + * instead of inheriting any policy from this new file's parent directory. + * This policy will be persisted in HDFS. A value of null means inheriting + * parent groups' whatever policy. + */ + public DFSOutputStream create(String src, FsPermission permission, + EnumSet flag, boolean createParent, short replication, + long blockSize, Progressable progress, int buffersize, + ChecksumOpt checksumOpt, InetSocketAddress[] favoredNodes, + String ecPolicyName) throws IOException { checkOpen(); final FsPermission masked = applyUMask(permission); LOG.debug("{}: masked={}", src, masked); final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this, src, masked, flag, createParent, replication, blockSize, progress, dfsClientConf.createChecksum(checksumOpt), - getFavoredNodesStr(favoredNodes)); + getFavoredNodesStr(favoredNodes), ecPolicyName); beginFileLease(result.getFileId(), result); return result; } @@ -1244,7 +1267,8 @@ public DFSOutputStream primitiveCreate(String src, FsPermission absPermission, if (result == null) { DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt); result = DFSOutputStream.newStreamForCreate(this, src, absPermission, - flag, createParent, replication, blockSize, progress, checksum, null); + flag, createParent, replication, blockSize, progress, checksum, + null, null); } beginFileLease(result.getFileId(), result); return result; @@ -2851,8 +2875,66 @@ DFSHedgedReadMetrics getHedgedReadMetrics() { return HEDGED_READ_METRIC; } - public KeyProvider getKeyProvider() { - return clientContext.getKeyProviderCache().get(conf); + /** + * Returns a key to map namenode uri to key provider uri. + * Tasks will lookup this key to find key Provider. + */ + public Text getKeyProviderMapKey() { + return new Text(DFS_KMS_PREFIX + namenodeUri.getScheme() + +"://" + namenodeUri.getAuthority()); + } + + /** + * The key provider uri is searched in the following order. + * 1. If there is a mapping in Credential's secrets map for namenode uri. + * 2. From namenode getServerDefaults rpc. + * 3. Finally fallback to local conf. + * @return keyProviderUri if found from either of above 3 cases, + * null otherwise + * @throws IOException + */ + URI getKeyProviderUri() throws IOException { + if (keyProviderUri != null) { + return keyProviderUri; + } + + // Lookup the secret in credentials object for namenodeuri. + Credentials credentials = ugi.getCredentials(); + byte[] keyProviderUriBytes = credentials.getSecretKey(getKeyProviderMapKey()); + if(keyProviderUriBytes != null) { + keyProviderUri = + URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes)); + return keyProviderUri; + } + + // Query the namenode for the key provider uri. + FsServerDefaults serverDefaults = getServerDefaults(); + if (serverDefaults.getKeyProviderUri() != null) { + if (!serverDefaults.getKeyProviderUri().isEmpty()) { + keyProviderUri = URI.create(serverDefaults.getKeyProviderUri()); + } + return keyProviderUri; + } + + // Last thing is to trust its own conf to be backwards compatible. + String keyProviderUriStr = conf.getTrimmed( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); + if (keyProviderUriStr != null && !keyProviderUriStr.isEmpty()) { + keyProviderUri = URI.create(keyProviderUriStr); + } + return keyProviderUri; + } + + public KeyProvider getKeyProvider() throws IOException { + return clientContext.getKeyProviderCache().get(conf, getKeyProviderUri()); + } + + /* + * Should be used only for testing. + */ + @VisibleForTesting + public void setKeyProviderUri(URI providerUri) { + this.keyProviderUri = providerUri; } @VisibleForTesting @@ -2862,11 +2944,10 @@ public void setKeyProvider(KeyProvider provider) { /** * Probe for encryption enabled on this filesystem. - * See {@link DFSUtilClient#isHDFSEncryptionEnabled(Configuration)} * @return true if encryption is enabled */ - public boolean isHDFSEncryptionEnabled() { - return DFSUtilClient.isHDFSEncryptionEnabled(this.conf); + public boolean isHDFSEncryptionEnabled() throws IOException{ + return getKeyProviderUri() != null; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index e4929e16b3f..ceaefd80b14 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; +import org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; @@ -122,6 +123,7 @@ public class DFSOutputStream extends FSOutputSummer private final EnumSet addBlockFlags; protected final AtomicReference cachingStrategy; private FileEncryptionInfo fileEncryptionInfo; + private int writePacketSize; /** Use {@link ByteArrayManager} to create buffer for non-heartbeat packets.*/ protected DFSPacket createPacket(int packetSize, int chunksPerPkt, @@ -202,6 +204,8 @@ private DFSOutputStream(DFSClient dfsClient, String src, +"{}", src); } + initWritePacketSize(); + this.bytesPerChecksum = checksum.getBytesPerChecksum(); if (bytesPerChecksum <= 0) { throw new HadoopIllegalArgumentException( @@ -216,6 +220,21 @@ private DFSOutputStream(DFSClient dfsClient, String src, this.byteArrayManager = dfsClient.getClientContext().getByteArrayManager(); } + /** + * Ensures the configured writePacketSize never exceeds + * PacketReceiver.MAX_PACKET_SIZE. + */ + private void initWritePacketSize() { + writePacketSize = dfsClient.getConf().getWritePacketSize(); + if (writePacketSize > PacketReceiver.MAX_PACKET_SIZE) { + LOG.warn( + "Configured write packet exceeds {} bytes as max," + + " using {} bytes.", + PacketReceiver.MAX_PACKET_SIZE, PacketReceiver.MAX_PACKET_SIZE); + writePacketSize = PacketReceiver.MAX_PACKET_SIZE; + } + } + /** Construct a new output stream for creating a file. */ protected DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat, EnumSet flag, Progressable progress, @@ -236,7 +255,8 @@ protected DFSOutputStream(DFSClient dfsClient, String src, static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, FsPermission masked, EnumSet flag, boolean createParent, short replication, long blockSize, Progressable progress, - DataChecksum checksum, String[] favoredNodes) throws IOException { + DataChecksum checksum, String[] favoredNodes, String ecPolicyName) + throws IOException { try (TraceScope ignored = dfsClient.newPathTraceScope("newStreamForCreate", src)) { HdfsFileStatus stat = null; @@ -250,7 +270,7 @@ static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, try { stat = dfsClient.namenode.create(src, masked, dfsClient.clientName, new EnumSetWritable<>(flag), createParent, replication, - blockSize, SUPPORTED_CRYPTO_VERSIONS); + blockSize, SUPPORTED_CRYPTO_VERSIONS, ecPolicyName); break; } catch (RemoteException re) { IOException e = re.unwrapRemoteException( @@ -489,12 +509,28 @@ protected void adjustChunkBoundary() { } if (!getStreamer().getAppendChunk()) { - int psize = Math.min((int)(blockSize- getStreamer().getBytesCurBlock()), - dfsClient.getConf().getWritePacketSize()); + final int psize = (int) Math + .min(blockSize - getStreamer().getBytesCurBlock(), writePacketSize); computePacketChunkSize(psize, bytesPerChecksum); } } + /** + * Used in test only. + */ + @VisibleForTesting + void setAppendChunk(final boolean appendChunk) { + getStreamer().setAppendChunk(appendChunk); + } + + /** + * Used in test only. + */ + @VisibleForTesting + void setBytesCurBlock(final long bytesCurBlock) { + getStreamer().setBytesCurBlock(bytesCurBlock); + } + /** * if encountering a block boundary, send an empty packet to * indicate the end of block and reset bytesCurBlock. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index f9b2e8d7c8a..2e770cc1c73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -169,6 +169,19 @@ static String addSuffix(String key, String suffix) { return key + "." + suffix; } + /** + * Returns list of InetSocketAddress corresponding to HA NN RPC addresses from + * the configuration. + * + * @param conf configuration + * @return list of InetSocketAddresses + */ + public static Map> getHaNnRpcAddresses( + Configuration conf) { + return DFSUtilClient.getAddresses(conf, null, + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY); + } + /** * Returns list of InetSocketAddress corresponding to HA NN HTTP addresses from * the configuration. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 5cd956cf346..11d7eb89771 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -100,6 +100,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import javax.annotation.Nonnull; + /**************************************************************** * Implementation of the abstract FileSystem for the DFS system. * This object is the way end-user code interacts with a Hadoop @@ -456,13 +458,18 @@ public FSDataOutputStream next(final FileSystem fs, final Path p) * at the creation time only. And with favored nodes, blocks will be pinned * on the datanodes to prevent balancing move the block. HDFS could move the * blocks during replication, to move the blocks from favored nodes. A value - * of null means no favored nodes for this create + * of null means no favored nodes for this create. + * Another addition is ecPolicyName. A non-null ecPolicyName specifies an + * explicit erasure coding policy for this file, overriding the inherited + * policy. A null ecPolicyName means the file will inherit its EC policy from + * an ancestor (the default). */ private HdfsDataOutputStream create(final Path f, - final FsPermission permission, EnumSet flag, + final FsPermission permission, final EnumSet flag, final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt, - final InetSocketAddress[] favoredNodes) throws IOException { + final InetSocketAddress[] favoredNodes, final String ecPolicyName) + throws IOException { statistics.incrementWriteOps(1); storageStatistics.incrementOpCounter(OpType.CREATE); Path absF = fixRelativePart(f); @@ -471,7 +478,7 @@ private HdfsDataOutputStream create(final Path f, public HdfsDataOutputStream doCall(final Path p) throws IOException { final DFSOutputStream out = dfs.create(getPathName(f), permission, flag, true, replication, blockSize, progress, bufferSize, - checksumOpt, favoredNodes); + checksumOpt, favoredNodes, ecPolicyName); return dfs.createWrappedOutputStream(out, statistics); } @Override @@ -480,7 +487,7 @@ public HdfsDataOutputStream next(final FileSystem fs, final Path p) if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem)fs; return myDfs.create(p, permission, flag, bufferSize, replication, - blockSize, progress, checksumOpt, favoredNodes); + blockSize, progress, checksumOpt, favoredNodes, ecPolicyName); } throw new UnsupportedOperationException("Cannot create with" + " favoredNodes through a symlink to a non-DistributedFileSystem: " @@ -2409,12 +2416,15 @@ public Void next(final FileSystem fs, final Path p) public Token[] addDelegationTokens( final String renewer, Credentials credentials) throws IOException { Token[] tokens = super.addDelegationTokens(renewer, credentials); - if (dfs.isHDFSEncryptionEnabled()) { + URI keyProviderUri = dfs.getKeyProviderUri(); + if (keyProviderUri != null) { KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension = KeyProviderDelegationTokenExtension. createKeyProviderDelegationTokenExtension(dfs.getKeyProvider()); Token[] kpTokens = keyProviderDelegationTokenExtension. addDelegationTokens(renewer, credentials); + credentials.addSecretKey(dfs.getKeyProviderMapKey(), + DFSUtilClient.string2Bytes(keyProviderUri.toString())); if (tokens != null && kpTokens != null) { Token[] all = new Token[tokens.length + kpTokens.length]; System.arraycopy(tokens, 0, all, 0, tokens.length); @@ -2551,7 +2561,13 @@ public Void next(final FileSystem fs, final Path p) throws IOException { */ @Override public Path getTrashRoot(Path path) { - if ((path == null) || !dfs.isHDFSEncryptionEnabled()) { + try { + if ((path == null) || !dfs.isHDFSEncryptionEnabled()) { + return super.getTrashRoot(path); + } + } catch (IOException ioe) { + DFSClient.LOG.warn("Exception while checking whether encryption zone is " + + "supported", ioe); return super.getTrashRoot(path); } @@ -2636,6 +2652,7 @@ public static class HdfsDataOutputStreamBuilder extends FSDataOutputStreamBuilder { private final DistributedFileSystem dfs; private InetSocketAddress[] favoredNodes = null; + private String ecPolicyName = null; public HdfsDataOutputStreamBuilder(DistributedFileSystem dfs, Path path) { super(dfs, path); @@ -2647,17 +2664,29 @@ protected InetSocketAddress[] getFavoredNodes() { } public HdfsDataOutputStreamBuilder setFavoredNodes( - final InetSocketAddress[] nodes) { + @Nonnull final InetSocketAddress[] nodes) { Preconditions.checkNotNull(nodes); favoredNodes = nodes.clone(); return this; } + protected String getEcPolicyName() { + return ecPolicyName; + } + + public HdfsDataOutputStreamBuilder setEcPolicyName( + @Nonnull final String policyName) { + Preconditions.checkNotNull(policyName); + ecPolicyName = policyName; + return this; + } + @Override public HdfsDataOutputStream build() throws IOException { return dfs.create(getPath(), getPermission(), getFlags(), getBufferSize(), getReplication(), getBlockSize(), - getProgress(), getChecksumOpt(), getFavoredNodes()); + getProgress(), getChecksumOpt(), getFavoredNodes(), + getEcPolicyName()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HAUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HAUtilClient.java index 9f28cfcde7c..47288f77df8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HAUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HAUtilClient.java @@ -20,15 +20,29 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.net.InetSocketAddress; import java.net.URI; +import java.util.Collection; import static org.apache.hadoop.hdfs.protocol.HdfsConstants.HA_DT_SERVICE_PREFIX; +import static org.apache.hadoop.security.SecurityUtil.buildTokenService; @InterfaceAudience.Private public class HAUtilClient { + private static final Logger LOG = LoggerFactory.getLogger(HAUtilClient.class); + + private static final DelegationTokenSelector tokenSelector = + new DelegationTokenSelector(); + /** * @return true if the given nameNodeUri appears to be a logical URI. */ @@ -92,4 +106,45 @@ public static URI getServiceUriFromToken(final String scheme, Token token) { public static boolean isTokenForLogicalUri(Token token) { return token.getService().toString().startsWith(HA_DT_SERVICE_PREFIX); } + + /** + * Locate a delegation token associated with the given HA cluster URI, and if + * one is found, clone it to also represent the underlying namenode address. + * @param ugi the UGI to modify + * @param haUri the logical URI for the cluster + * @param nnAddrs collection of NNs in the cluster to which the token + * applies + */ + public static void cloneDelegationTokenForLogicalUri( + UserGroupInformation ugi, URI haUri, + Collection nnAddrs) { + // this cloning logic is only used by hdfs + Text haService = HAUtilClient.buildTokenServiceForLogicalUri(haUri, + HdfsConstants.HDFS_URI_SCHEME); + Token haToken = + tokenSelector.selectToken(haService, ugi.getTokens()); + if (haToken != null) { + for (InetSocketAddress singleNNAddr : nnAddrs) { + // this is a minor hack to prevent physical HA tokens from being + // exposed to the user via UGI.getCredentials(), otherwise these + // cloned tokens may be inadvertently propagated to jobs + Token specificToken = + haToken.privateClone(buildTokenService(singleNNAddr)); + Text alias = new Text( + HAUtilClient.buildTokenServicePrefixForLogicalUri( + HdfsConstants.HDFS_URI_SCHEME) + + "//" + specificToken.getService()); + ugi.addToken(alias, specificToken); + if (LOG.isDebugEnabled()) { + LOG.debug("Mapped HA service delegation token for logical URI " + + haUri + " to namenode " + singleNNAddr); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No HA service delegation token found for logical URI " + + haUri); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java index 3491010aa1b..17d20fe014d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.util.KMSUtil; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; @@ -66,29 +67,29 @@ public void onRemoval( .build(); } - public KeyProvider get(final Configuration conf) { - URI kpURI = createKeyProviderURI(conf); - if (kpURI == null) { + public KeyProvider get(final Configuration conf, + final URI serverProviderUri) { + if (serverProviderUri == null) { return null; } try { - return cache.get(kpURI, new Callable() { + return cache.get(serverProviderUri, new Callable() { @Override public KeyProvider call() throws Exception { - return DFSUtilClient.createKeyProvider(conf); + return KMSUtil.createKeyProviderFromUri(conf, serverProviderUri); } }); } catch (Exception e) { - LOG.error("Could not create KeyProvider for DFSClient !!", e.getCause()); + LOG.error("Could not create KeyProvider for DFSClient !!", e); return null; } } private URI createKeyProviderURI(Configuration conf) { final String providerUriStr = conf.getTrimmed( - CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, ""); + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); // No provider set in conf - if (providerUriStr.isEmpty()) { + if (providerUriStr == null || providerUriStr.isEmpty()) { LOG.error("Could not find uri with key [" + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH + "] to create a keyProvider !!"); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java index 5ca7030acdd..a092f02630d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java @@ -28,6 +28,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.server.namenode.ha.ClientHAProxyFactory; +import org.apache.hadoop.hdfs.server.namenode.ha.HAProxyFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -212,6 +214,14 @@ public static ProxyAndInfo createProxyWithLossyRetryHandler( public static AbstractNNFailoverProxyProvider createFailoverProxyProvider( Configuration conf, URI nameNodeUri, Class xface, boolean checkPort, AtomicBoolean fallbackToSimpleAuth) throws IOException { + return createFailoverProxyProvider(conf, nameNodeUri, xface, checkPort, + fallbackToSimpleAuth, new ClientHAProxyFactory()); + } + + protected static AbstractNNFailoverProxyProvider createFailoverProxyProvider( + Configuration conf, URI nameNodeUri, Class xface, boolean checkPort, + AtomicBoolean fallbackToSimpleAuth, HAProxyFactory proxyFactory) + throws IOException { Class> failoverProxyProviderClass = null; AbstractNNFailoverProxyProvider providerNN; try { @@ -223,9 +233,10 @@ public static AbstractNNFailoverProxyProvider createFailoverProxyProvider } // Create a proxy provider instance. Constructor> ctor = failoverProxyProviderClass - .getConstructor(Configuration.class, URI.class, Class.class); + .getConstructor(Configuration.class, URI.class, + Class.class, HAProxyFactory.class); FailoverProxyProvider provider = ctor.newInstance(conf, nameNodeUri, - xface); + xface, proxyFactory); // If the proxy provider is of an old implementation, wrap it. if (!(provider instanceof AbstractNNFailoverProxyProvider)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 1a388061ceb..c152a4bf3cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -67,6 +67,7 @@ public interface HdfsClientConfigKeys { String PREFIX = "dfs.client."; String DFS_NAMESERVICES = "dfs.nameservices"; + String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address"; int DFS_NAMENODE_HTTP_PORT_DEFAULT = 9870; String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address"; int DFS_NAMENODE_HTTPS_PORT_DEFAULT = 9871; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 041d2268402..6db37b8f05f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -160,6 +160,10 @@ LocatedBlocks getBlockLocations(String src, long offset, long length) * @param replication block replication factor. * @param blockSize maximum block size. * @param supportedVersions CryptoProtocolVersions supported by the client + * @param ecPolicyName the name of erasure coding policy. A null value means + * this file will inherit its parent directory's policy, + * either traditional replication or erasure coding + * policy. * * @return the status of the created file, it could be null if the server * doesn't support returning the file status @@ -193,7 +197,7 @@ LocatedBlocks getBlockLocations(String src, long offset, long length) HdfsFileStatus create(String src, FsPermission masked, String clientName, EnumSetWritable flag, boolean createParent, short replication, long blockSize, - CryptoProtocolVersion[] supportedVersions) + CryptoProtocolVersion[] supportedVersions, String ecPolicyName) throws IOException; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index d2209a49952..0d31bc40fa2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -144,12 +144,6 @@ public enum DatanodeReportType { ALL, LIVE, DEAD, DECOMMISSIONING, ENTERING_MAINTENANCE } - public static final byte RS_6_3_POLICY_ID = 1; - public static final byte RS_3_2_POLICY_ID = 2; - public static final byte RS_6_3_LEGACY_POLICY_ID = 3; - public static final byte XOR_2_1_POLICY_ID = 4; - public static final byte RS_10_4_POLICY_ID = 5; - /* Hidden constructor */ protected HdfsConstants() { } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java new file mode 100644 index 00000000000..2cd838b8fa7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + *

The set of built-in erasure coding policies.

+ *

Although this is a private class, EC policy IDs need to be treated like a + * stable interface. Adding, modifying, or removing built-in policies can cause + * inconsistencies with older clients.

+ */ +@InterfaceAudience.Private +@InterfaceStability.Stable +public final class SystemErasureCodingPolicies { + + // Private constructor, this is a utility class. + private SystemErasureCodingPolicies() {} + + // 64 KB + private static final int DEFAULT_CELLSIZE = 64 * 1024; + + public static final byte RS_6_3_POLICY_ID = 1; + private static final ErasureCodingPolicy SYS_POLICY1 = + new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_SCHEMA, + DEFAULT_CELLSIZE, RS_6_3_POLICY_ID); + + public static final byte RS_3_2_POLICY_ID = 2; + private static final ErasureCodingPolicy SYS_POLICY2 = + new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, + DEFAULT_CELLSIZE, RS_3_2_POLICY_ID); + + public static final byte RS_6_3_LEGACY_POLICY_ID = 3; + private static final ErasureCodingPolicy SYS_POLICY3 = + new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA, + DEFAULT_CELLSIZE, RS_6_3_LEGACY_POLICY_ID); + + public static final byte XOR_2_1_POLICY_ID = 4; + private static final ErasureCodingPolicy SYS_POLICY4 = + new ErasureCodingPolicy(ErasureCodeConstants.XOR_2_1_SCHEMA, + DEFAULT_CELLSIZE, XOR_2_1_POLICY_ID); + + public static final byte RS_10_4_POLICY_ID = 5; + private static final ErasureCodingPolicy SYS_POLICY5 = + new ErasureCodingPolicy(ErasureCodeConstants.RS_10_4_SCHEMA, + DEFAULT_CELLSIZE, RS_10_4_POLICY_ID); + + private static final List SYS_POLICIES = + Collections.unmodifiableList(Arrays.asList( + SYS_POLICY1, SYS_POLICY2, SYS_POLICY3, SYS_POLICY4, + SYS_POLICY5)); + + /** + * System policies sorted by name for fast querying. + */ + private static final Map SYSTEM_POLICIES_BY_NAME; + + /** + * System policies sorted by ID for fast querying. + */ + private static final Map SYSTEM_POLICIES_BY_ID; + + /** + * Populate the lookup maps in a static block. + */ + static { + SYSTEM_POLICIES_BY_NAME = new TreeMap<>(); + SYSTEM_POLICIES_BY_ID = new TreeMap<>(); + for (ErasureCodingPolicy policy : SYS_POLICIES) { + SYSTEM_POLICIES_BY_NAME.put(policy.getName(), policy); + SYSTEM_POLICIES_BY_ID.put(policy.getId(), policy); + } + } + + /** + * Get system defined policies. + * @return system policies + */ + public static List getPolicies() { + return SYS_POLICIES; + } + + /** + * Get a policy by policy ID. + * @return ecPolicy, or null if not found + */ + public static ErasureCodingPolicy getByID(byte id) { + return SYSTEM_POLICIES_BY_ID.get(id); + } + + /** + * Get a policy by policy name. + * @return ecPolicy, or null if not found + */ + public static ErasureCodingPolicy getByName(String name) { + return SYSTEM_POLICIES_BY_NAME.get(name); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java index 93e9217fbb2..6b717ecdfbf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java @@ -45,7 +45,7 @@ public class PacketReceiver implements Closeable { * The max size of any single packet. This prevents OOMEs when * invalid data is sent. */ - private static final int MAX_PACKET_SIZE = 16 * 1024 * 1024; + public static final int MAX_PACKET_SIZE = 16 * 1024 * 1024; static final Logger LOG = LoggerFactory.getLogger(PacketReceiver.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index da4a17f5799..c3708f98785 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -285,7 +285,7 @@ public FsServerDefaults getServerDefaults() throws IOException { public HdfsFileStatus create(String src, FsPermission masked, String clientName, EnumSetWritable flag, boolean createParent, short replication, long blockSize, - CryptoProtocolVersion[] supportedVersions) + CryptoProtocolVersion[] supportedVersions, String ecPolicyName) throws IOException { CreateRequestProto.Builder builder = CreateRequestProto.newBuilder() .setSrc(src) @@ -295,6 +295,9 @@ public HdfsFileStatus create(String src, FsPermission masked, .setCreateParent(createParent) .setReplication(replication) .setBlockSize(blockSize); + if (ecPolicyName != null) { + builder.setEcPolicyName(ecPolicyName); + } FsPermission unmasked = masked.getUnmasked(); if (unmasked != null) { builder.setUnmasked(PBHelperClient.convert(unmasked)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 946a5cba7ba..e645ddac72b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -94,6 +94,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto; @@ -1760,7 +1761,9 @@ public static FsServerDefaults convert(FsServerDefaultsProto fs) { fs.getFileBufferSize(), fs.getEncryptDataTransfer(), fs.getTrashInterval(), - convert(fs.getChecksumType())); + convert(fs.getChecksumType()), + fs.hasKeyProviderUri() ? fs.getKeyProviderUri() : null, + (byte) fs.getPolicyId()); } public static List convert( @@ -1934,6 +1937,8 @@ public static FsServerDefaultsProto convert(FsServerDefaults fs) { .setEncryptDataTransfer(fs.getEncryptDataTransfer()) .setTrashInterval(fs.getTrashInterval()) .setChecksumType(convert(fs.getChecksumType())) + .setKeyProviderUri(fs.getKeyProviderUri()) + .setPolicyId(fs.getDefaultStoragePolicyId()) .build(); } @@ -2652,20 +2657,37 @@ public static HdfsProtos.ECSchemaProto convertECSchema(ECSchema schema) { } public static ErasureCodingPolicy convertErasureCodingPolicy( - ErasureCodingPolicyProto policy) { - return new ErasureCodingPolicy(policy.getName(), - convertECSchema(policy.getSchema()), - policy.getCellSize(), (byte) policy.getId()); + ErasureCodingPolicyProto proto) { + final byte id = (byte) (proto.getId() & 0xFF); + ErasureCodingPolicy policy = SystemErasureCodingPolicies.getByID(id); + if (policy == null) { + // If it's not a built-in policy, populate from the optional PB fields. + // The optional fields are required in this case. + Preconditions.checkArgument(proto.hasName(), + "Missing name field in ErasureCodingPolicy proto"); + Preconditions.checkArgument(proto.hasSchema(), + "Missing schema field in ErasureCodingPolicy proto"); + Preconditions.checkArgument(proto.hasCellSize(), + "Missing cellsize field in ErasureCodingPolicy proto"); + + return new ErasureCodingPolicy(proto.getName(), + convertECSchema(proto.getSchema()), + proto.getCellSize(), id); + } + return policy; } public static ErasureCodingPolicyProto convertErasureCodingPolicy( ErasureCodingPolicy policy) { ErasureCodingPolicyProto.Builder builder = ErasureCodingPolicyProto .newBuilder() - .setName(policy.getName()) - .setSchema(convertECSchema(policy.getSchema())) - .setCellSize(policy.getCellSize()) .setId(policy.getId()); + // If it's not a built-in policy, need to set the optional fields. + if (SystemErasureCodingPolicies.getByID(policy.getId()) == null) { + builder.setName(policy.getName()) + .setSchema(convertECSchema(policy.getSchema())) + .setCellSize(policy.getCellSize()); + } return builder.build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ClientHAProxyFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ClientHAProxyFactory.java new file mode 100644 index 00000000000..b887d87100e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ClientHAProxyFactory.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.ha; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.NameNodeProxiesClient; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.AtomicBoolean; + +public class ClientHAProxyFactory implements HAProxyFactory { + @Override + @SuppressWarnings("unchecked") + public T createProxy(Configuration conf, InetSocketAddress nnAddr, + Class xface, UserGroupInformation ugi, boolean withRetries, + AtomicBoolean fallbackToSimpleAuth) throws IOException { + return (T) NameNodeProxiesClient.createNonHAProxyWithClientProtocol( + nnAddr, conf, ugi, false, fallbackToSimpleAuth); + } + + @Override + public T createProxy(Configuration conf, InetSocketAddress nnAddr, + Class xface, UserGroupInformation ugi, boolean withRetries) + throws IOException { + return createProxy(conf, nnAddr, xface, ugi, withRetries, null); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java similarity index 76% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java index 0e8fa448806..e9c8791c5c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java @@ -26,22 +26,16 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdfs.HAUtil; -import org.apache.hadoop.hdfs.NameNodeProxies; +import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.HAUtilClient; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; -import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A FailoverProxyProvider implementation which allows one to configure @@ -51,25 +45,9 @@ */ public class ConfiguredFailoverProxyProvider extends AbstractNNFailoverProxyProvider { - - private static final Log LOG = - LogFactory.getLog(ConfiguredFailoverProxyProvider.class); - - interface ProxyFactory { - T createProxy(Configuration conf, InetSocketAddress nnAddr, Class xface, - UserGroupInformation ugi, boolean withRetries, - AtomicBoolean fallbackToSimpleAuth) throws IOException; - } - static class DefaultProxyFactory implements ProxyFactory { - @Override - public T createProxy(Configuration conf, InetSocketAddress nnAddr, - Class xface, UserGroupInformation ugi, boolean withRetries, - AtomicBoolean fallbackToSimpleAuth) throws IOException { - return NameNodeProxies.createNonHAProxy(conf, - nnAddr, xface, ugi, false, fallbackToSimpleAuth).getProxy(); - } - } + private static final Logger LOG = + LoggerFactory.getLogger(ConfiguredFailoverProxyProvider.class); protected final Configuration conf; protected final List> proxies = @@ -78,22 +56,11 @@ public T createProxy(Configuration conf, InetSocketAddress nnAddr, protected final Class xface; private int currentProxyIndex = 0; - private final ProxyFactory factory; + private final HAProxyFactory factory; public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, - Class xface) { - this(conf, uri, xface, new DefaultProxyFactory()); - } - - @VisibleForTesting - ConfiguredFailoverProxyProvider(Configuration conf, URI uri, - Class xface, ProxyFactory factory) { - - Preconditions.checkArgument( - xface.isAssignableFrom(NamenodeProtocols.class), - "Interface class %s is not a valid NameNode protocol!"); + Class xface, HAProxyFactory factory) { this.xface = xface; - this.conf = new Configuration(conf); int maxRetries = this.conf.getInt( HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_KEY, @@ -101,7 +68,7 @@ public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, this.conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, maxRetries); - + int maxRetriesOnSocketTimeouts = this.conf.getInt( HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY, HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT); @@ -112,16 +79,16 @@ public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, try { ugi = UserGroupInformation.getCurrentUser(); - - Map> map = DFSUtil.getHaNnRpcAddresses( - conf); + + Map> map = + DFSUtilClient.getHaNnRpcAddresses(conf); Map addressesInNN = map.get(uri.getHost()); - + if (addressesInNN == null || addressesInNN.size() == 0) { throw new RuntimeException("Could not find any configured addresses " + "for URI " + uri); } - + Collection addressesOfNns = addressesInNN.values(); for (InetSocketAddress address : addressesOfNns) { proxies.add(new AddressRpcProxyPair(address)); @@ -137,13 +104,13 @@ public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, // The client may have a delegation token set for the logical // URI of the cluster. Clone this token to apply to each of the // underlying IPC addresses so that the IPC code can find it. - HAUtil.cloneDelegationTokenForLogicalUri(ugi, uri, addressesOfNns); + HAUtilClient.cloneDelegationTokenForLogicalUri(ugi, uri, addressesOfNns); this.factory = factory; } catch (IOException e) { throw new RuntimeException(e); } } - + @Override public Class getInterface() { return xface; @@ -183,7 +150,7 @@ synchronized void incrementProxyIndex() { private static class AddressRpcProxyPair { public final InetSocketAddress address; public T namenode; - + public AddressRpcProxyPair(InetSocketAddress address) { this.address = address; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/HAProxyFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/HAProxyFactory.java new file mode 100644 index 00000000000..f92a74ff7ce --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/HAProxyFactory.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.ha; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * This interface aims to decouple the proxy creation implementation that used + * in {@link AbstractNNFailoverProxyProvider}. Client side can use + * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} to initialize the + * proxy while the server side can use NamenodeProtocols + */ +@InterfaceAudience.Private +public interface HAProxyFactory { + + T createProxy(Configuration conf, InetSocketAddress nnAddr, Class xface, + UserGroupInformation ugi, boolean withRetries, + AtomicBoolean fallbackToSimpleAuth) throws IOException; + + T createProxy(Configuration conf, InetSocketAddress nnAddr, Class xface, + UserGroupInformation ugi, boolean withRetries) throws IOException; + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java similarity index 87% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java index 4e1cb9e68dd..ed250a0f42e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java @@ -25,14 +25,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; -import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; -import com.google.common.base.Preconditions; - /** * A NNFailoverProxyProvider implementation which works on IP failover setup. * Only one proxy is used to connect to both servers and switching between @@ -40,7 +36,7 @@ * clients can consistently reach only one node at a time. * * Clients with a live connection will likely get connection reset after an - * IP failover. This case will be handled by the + * IP failover. This case will be handled by the * FailoverOnNetworkExceptionRetry retry policy. I.e. if the call is * not idempotent, it won't get retried. * @@ -54,15 +50,14 @@ public class IPFailoverProxyProvider extends private final Configuration conf; private final Class xface; private final URI nameNodeUri; + private final HAProxyFactory factory; private ProxyInfo nnProxyInfo = null; - + public IPFailoverProxyProvider(Configuration conf, URI uri, - Class xface) { - Preconditions.checkArgument( - xface.isAssignableFrom(NamenodeProtocols.class), - "Interface class %s is not a valid NameNode protocol!"); + Class xface, HAProxyFactory factory) { this.xface = xface; this.nameNodeUri = uri; + this.factory = factory; this.conf = new Configuration(conf); int maxRetries = this.conf.getInt( @@ -71,7 +66,7 @@ public IPFailoverProxyProvider(Configuration conf, URI uri, this.conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, maxRetries); - + int maxRetriesOnSocketTimeouts = this.conf.getInt( HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY, HdfsClientConfigKeys.Failover.CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT); @@ -79,7 +74,7 @@ public IPFailoverProxyProvider(Configuration conf, URI uri, CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, maxRetriesOnSocketTimeouts); } - + @Override public Class getInterface() { return xface; @@ -92,9 +87,8 @@ public synchronized ProxyInfo getProxy() { try { // Create a proxy that is not wrapped in RetryProxy InetSocketAddress nnAddr = DFSUtilClient.getNNAddress(nameNodeUri); - nnProxyInfo = new ProxyInfo(NameNodeProxies.createNonHAProxy( - conf, nnAddr, xface, UserGroupInformation.getCurrentUser(), - false).getProxy(), nnAddr.toString()); + nnProxyInfo = new ProxyInfo(factory.createProxy(conf, nnAddr, xface, + UserGroupInformation.getCurrentUser(), false), nnAddr.toString()); } catch (IOException ioe) { throw new RuntimeException(ioe); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java similarity index 95% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java index 2f6c9bc7097..b94e94d67bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java @@ -34,7 +34,6 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.io.retry.MultiException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,15 +146,9 @@ public Object call() throws Exception { private volatile ProxyInfo successfulProxy = null; private volatile String toIgnore = null; - public RequestHedgingProxyProvider( - Configuration conf, URI uri, Class xface) { - this(conf, uri, xface, new DefaultProxyFactory()); - } - - @VisibleForTesting - RequestHedgingProxyProvider(Configuration conf, URI uri, - Class xface, ProxyFactory factory) { - super(conf, uri, xface, factory); + public RequestHedgingProxyProvider(Configuration conf, URI uri, + Class xface, HAProxyFactory proxyFactory) { + super(conf, uri, xface, proxyFactory); } @SuppressWarnings("unchecked") diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index ff4db035904..44f1c3373b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -80,6 +80,7 @@ message CreateRequestProto { required uint64 blockSize = 7; repeated CryptoProtocolVersionProto cryptoProtocolVersion = 8; optional FsPermissionProto unmasked = 9; + optional string ecPolicyName = 10; } message CreateResponseProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index b12ee2f1c12..7b54938dfae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -353,9 +353,9 @@ message ECSchemaProto { } message ErasureCodingPolicyProto { - required string name = 1; - required ECSchemaProto schema = 2; - required uint32 cellSize = 3; + optional string name = 1; + optional ECSchemaProto schema = 2; + optional uint32 cellSize = 3; required uint32 id = 4; // Actually a byte - only 8 bits used } @@ -421,6 +421,8 @@ message FsServerDefaultsProto { optional bool encryptDataTransfer = 6 [default = false]; optional uint64 trashInterval = 7 [default = 0]; optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32]; + optional string keyProviderUri = 9; + optional uint32 policyId = 10 [default = 0]; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java similarity index 81% rename from hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java rename to hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java index 37532d5c880..724b5f01c4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java @@ -29,9 +29,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.ProxyFactory; -import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.io.retry.MultiException; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; @@ -66,20 +65,20 @@ public void setup() throws URISyntaxException { ns = "mycluster-" + Time.monotonicNow(); nnUri = new URI("hdfs://" + ns); conf = new Configuration(); - conf.set(DFSConfigKeys.DFS_NAMESERVICES, ns); + conf.set(HdfsClientConfigKeys.DFS_NAMESERVICES, ns); conf.set( - DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, "nn1,nn2"); + HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, "nn1,nn2"); conf.set( - DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns + ".nn1", + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns + ".nn1", "machine1.foo.bar:9820"); conf.set( - DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns + ".nn2", + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns + ".nn2", "machine2.foo.bar:9820"); } @Test public void testHedgingWhenOneFails() throws Exception { - final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol goodMock = Mockito.mock(ClientProtocol.class); Mockito.when(goodMock.getStats()).thenAnswer(new Answer() { @Override public long[] answer(InvocationOnMock invocation) throws Throwable { @@ -87,11 +86,11 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { return new long[]{1}; } }); - final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol badMock = Mockito.mock(ClientProtocol.class); Mockito.when(badMock.getStats()).thenThrow(new IOException("Bad mock !!")); - RequestHedgingProxyProvider provider = - new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class, createFactory(badMock, goodMock)); long[] stats = provider.getProxy().proxy.getStats(); Assert.assertTrue(stats.length == 1); @@ -101,7 +100,7 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { @Test public void testHedgingWhenOneIsSlow() throws Exception { - final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol goodMock = Mockito.mock(ClientProtocol.class); Mockito.when(goodMock.getStats()).thenAnswer(new Answer() { @Override public long[] answer(InvocationOnMock invocation) throws Throwable { @@ -109,11 +108,11 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { return new long[]{1}; } }); - final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol badMock = Mockito.mock(ClientProtocol.class); Mockito.when(badMock.getStats()).thenThrow(new IOException("Bad mock !!")); - RequestHedgingProxyProvider provider = - new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class, createFactory(goodMock, badMock)); long[] stats = provider.getProxy().proxy.getStats(); Assert.assertTrue(stats.length == 1); @@ -124,14 +123,14 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { @Test public void testHedgingWhenBothFail() throws Exception { - NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class); + ClientProtocol badMock = Mockito.mock(ClientProtocol.class); Mockito.when(badMock.getStats()).thenThrow(new IOException("Bad mock !!")); - NamenodeProtocols worseMock = Mockito.mock(NamenodeProtocols.class); + ClientProtocol worseMock = Mockito.mock(ClientProtocol.class); Mockito.when(worseMock.getStats()).thenThrow( new IOException("Worse mock !!")); - RequestHedgingProxyProvider provider = - new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class, createFactory(badMock, worseMock)); try { provider.getProxy().proxy.getStats(); @@ -147,7 +146,7 @@ public void testHedgingWhenBothFail() throws Exception { public void testPerformFailover() throws Exception { final AtomicInteger counter = new AtomicInteger(0); final int[] isGood = {1}; - final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol goodMock = Mockito.mock(ClientProtocol.class); Mockito.when(goodMock.getStats()).thenAnswer(new Answer() { @Override public long[] answer(InvocationOnMock invocation) throws Throwable { @@ -159,7 +158,7 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { throw new IOException("Was Good mock !!"); } }); - final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol badMock = Mockito.mock(ClientProtocol.class); Mockito.when(badMock.getStats()).thenAnswer(new Answer() { @Override public long[] answer(InvocationOnMock invocation) throws Throwable { @@ -171,8 +170,8 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { throw new IOException("Bad mock !!"); } }); - RequestHedgingProxyProvider provider = - new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class, createFactory(goodMock, badMock)); long[] stats = provider.getProxy().proxy.getStats(); Assert.assertTrue(stats.length == 1); @@ -234,14 +233,14 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { @Test public void testPerformFailoverWith3Proxies() throws Exception { - conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, + conf.set(HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns, "nn1,nn2,nn3"); - conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns + ".nn3", + conf.set(HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns + ".nn3", "machine3.foo.bar:9820"); final AtomicInteger counter = new AtomicInteger(0); final int[] isGood = {1}; - final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol goodMock = Mockito.mock(ClientProtocol.class); Mockito.when(goodMock.getStats()).thenAnswer(new Answer() { @Override public long[] answer(InvocationOnMock invocation) throws Throwable { @@ -253,7 +252,7 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { throw new IOException("Was Good mock !!"); } }); - final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol badMock = Mockito.mock(ClientProtocol.class); Mockito.when(badMock.getStats()).thenAnswer(new Answer() { @Override public long[] answer(InvocationOnMock invocation) throws Throwable { @@ -265,7 +264,7 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { throw new IOException("Bad mock !!"); } }); - final NamenodeProtocols worseMock = Mockito.mock(NamenodeProtocols.class); + final ClientProtocol worseMock = Mockito.mock(ClientProtocol.class); Mockito.when(worseMock.getStats()).thenAnswer(new Answer() { @Override public long[] answer(InvocationOnMock invocation) throws Throwable { @@ -278,8 +277,8 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { } }); - RequestHedgingProxyProvider provider = - new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class, createFactory(goodMock, badMock, worseMock)); long[] stats = provider.getProxy().proxy.getStats(); Assert.assertTrue(stats.length == 1); @@ -355,14 +354,14 @@ public long[] answer(InvocationOnMock invocation) throws Throwable { @Test public void testHedgingWhenFileNotFoundException() throws Exception { - NamenodeProtocols active = Mockito.mock(NamenodeProtocols.class); + ClientProtocol active = Mockito.mock(ClientProtocol.class); Mockito .when(active.getBlockLocations(Matchers.anyString(), Matchers.anyLong(), Matchers.anyLong())) .thenThrow(new RemoteException("java.io.FileNotFoundException", "File does not exist!")); - NamenodeProtocols standby = Mockito.mock(NamenodeProtocols.class); + ClientProtocol standby = Mockito.mock(ClientProtocol.class); Mockito .when(standby.getBlockLocations(Matchers.anyString(), Matchers.anyLong(), Matchers.anyLong())) @@ -370,9 +369,9 @@ public void testHedgingWhenFileNotFoundException() throws Exception { new RemoteException("org.apache.hadoop.ipc.StandbyException", "Standby NameNode")); - RequestHedgingProxyProvider provider = + RequestHedgingProxyProvider provider = new RequestHedgingProxyProvider<>(conf, nnUri, - NamenodeProtocols.class, createFactory(active, standby)); + ClientProtocol.class, createFactory(active, standby)); try { provider.getProxy().proxy.getBlockLocations("/tmp/test.file", 0L, 20L); Assert.fail("Should fail since the active namenode throws" @@ -394,18 +393,18 @@ public void testHedgingWhenFileNotFoundException() throws Exception { @Test public void testHedgingWhenConnectException() throws Exception { - NamenodeProtocols active = Mockito.mock(NamenodeProtocols.class); + ClientProtocol active = Mockito.mock(ClientProtocol.class); Mockito.when(active.getStats()).thenThrow(new ConnectException()); - NamenodeProtocols standby = Mockito.mock(NamenodeProtocols.class); + ClientProtocol standby = Mockito.mock(ClientProtocol.class); Mockito.when(standby.getStats()) .thenThrow( new RemoteException("org.apache.hadoop.ipc.StandbyException", "Standby NameNode")); - RequestHedgingProxyProvider provider = + RequestHedgingProxyProvider provider = new RequestHedgingProxyProvider<>(conf, nnUri, - NamenodeProtocols.class, createFactory(active, standby)); + ClientProtocol.class, createFactory(active, standby)); try { provider.getProxy().proxy.getStats(); Assert.fail("Should fail since the active namenode throws" @@ -428,15 +427,15 @@ public void testHedgingWhenConnectException() throws Exception { @Test public void testHedgingWhenConnectAndEOFException() throws Exception { - NamenodeProtocols active = Mockito.mock(NamenodeProtocols.class); + ClientProtocol active = Mockito.mock(ClientProtocol.class); Mockito.when(active.getStats()).thenThrow(new EOFException()); - NamenodeProtocols standby = Mockito.mock(NamenodeProtocols.class); + ClientProtocol standby = Mockito.mock(ClientProtocol.class); Mockito.when(standby.getStats()).thenThrow(new ConnectException()); - RequestHedgingProxyProvider provider = + RequestHedgingProxyProvider provider = new RequestHedgingProxyProvider<>(conf, nnUri, - NamenodeProtocols.class, createFactory(active, standby)); + ClientProtocol.class, createFactory(active, standby)); try { provider.getProxy().proxy.getStats(); Assert.fail("Should fail since both active and standby namenodes throw" @@ -453,18 +452,25 @@ public void testHedgingWhenConnectAndEOFException() throws Exception { Mockito.verify(standby).getStats(); } - private ProxyFactory createFactory( - NamenodeProtocols... protos) { - final Iterator iterator = + private HAProxyFactory createFactory( + ClientProtocol... protos) { + final Iterator iterator = Lists.newArrayList(protos).iterator(); - return new ProxyFactory() { + return new HAProxyFactory() { @Override - public NamenodeProtocols createProxy(Configuration conf, - InetSocketAddress nnAddr, Class xface, + public ClientProtocol createProxy(Configuration conf, + InetSocketAddress nnAddr, Class xface, UserGroupInformation ugi, boolean withRetries, AtomicBoolean fallbackToSimpleAuth) throws IOException { return iterator.next(); } + + @Override + public ClientProtocol createProxy(Configuration conf, + InetSocketAddress nnAddr, Class xface, + UserGroupInformation ugi, boolean withRetries) throws IOException { + return iterator.next(); + } }; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index f5cfd015679..a05304b6137 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -171,6 +171,11 @@ + + org.apache.hadoop + hadoop-hdfs-client + provided + org.apache.hadoop hadoop-common diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java index 258dde54f82..251193a9581 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java @@ -31,9 +31,8 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.junit.Test; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.Statement; @@ -142,7 +141,7 @@ public static Configuration getHdfsConf() { public static final Path ERASURE_CODING_DIR = new Path("/ec"); public static final Path ERASURE_CODING_FILE = new Path("/ec/ecfile"); public static final ErasureCodingPolicy ERASURE_CODING_POLICY = - ErasureCodingPolicyManager.getPolicyByID(HdfsConstants.XOR_2_1_POLICY_ID); + StripedFileTestUtil.getDefaultECPolicy(); private static MiniDFSCluster MINI_DFS = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index aceb5bcb777..65416698769 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -55,6 +55,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs compile + + org.apache.hadoop + hadoop-hdfs-client + provided + org.apache.hadoop hadoop-hdfs @@ -163,11 +168,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> slf4j-log4j12 provided - - xmlenc - xmlenc - compile - org.bouncycastle bcprov-jdk16 diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 7df8bf4dbab..20814002e88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -60,7 +60,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop hadoop-hdfs-client - compile + provided org.apache.zookeeper @@ -163,11 +163,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> slf4j-log4j12 provided - - xmlenc - xmlenc - compile - io.netty netty @@ -396,6 +391,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> src/test/resources/editsStored* src/test/resources/empty-file src/main/webapps/datanode/robots.txt + src/main/webapps/hdfs/robots.txt + src/main/webapps/journal/robots.txt + src/main/webapps/secondary/robots.txt src/contrib/** src/site/resources/images/* src/main/webapps/static/bootstrap-3.0.2/** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index be39d321e68..560de9439a9 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -93,20 +93,10 @@ function hdfscmd_case ;; datanode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - # Determine if we're starting a secure datanode, and - # if so, redefine appropriate variables - if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then - HADOOP_SUBCMD_SECURESERVICE="true" - HADOOP_SUBCMD_SECUREUSER="${HADOOP_SECURE_DN_USER}" - - # backward compatiblity - HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}" - HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}" - - HADOOP_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter" - else - HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode' - fi + HADOOP_SECURE_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter" + HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode' + hadoop_deprecate_envvar HADOOP_SECURE_DN_PID_DIR HADOOP_SECURE_PID_DIR + hadoop_deprecate_envvar HADOOP_SECURE_DN_LOG_DIR HADOOP_SECURE_LOG_DIR ;; debug) HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DebugAdmin' @@ -170,18 +160,10 @@ function hdfscmd_case ;; nfs3) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then - HADOOP_SUBCMD_SECURESERVICE="true" - HADOOP_SUBCMD_SECUREUSER="${HADOOP_PRIVILEGED_NFS_USER}" - - # backward compatiblity - HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}" - HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}" - - HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter - else - HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 - fi + HADOOP_SECURE_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter + HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 + hadoop_deprecate_envvar HADOOP_SECURE_NFS3_LOG_DIR HADOOP_SECURE_LOG_DIR + hadoop_deprecate_envvar HADOOP_SECURE_NFS3_PID_DIR HADOOP_SECURE_PID_DIR ;; oev) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer @@ -241,9 +223,9 @@ else fi HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" -# shellcheck disable=SC2034 HADOOP_NEW_CONFIG=true if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + # shellcheck source=./hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" else echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 @@ -268,7 +250,7 @@ if hadoop_need_reexec hdfs "${HADOOP_SUBCMD}"; then exit $? fi -hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" +hadoop_verify_user_perm "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@") @@ -288,60 +270,5 @@ fi hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" -if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then - HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}" - - hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" - - hadoop_verify_secure_prereq - hadoop_setup_secure_service - priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err" - priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" - daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" -else - daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" -fi - -if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then - # shellcheck disable=SC2034 - HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" - if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then - # shellcheck disable=SC2034 - HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log" - else - # shellcheck disable=SC2034 - HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log" - fi -fi - -hadoop_finalize - -if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then - if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then - hadoop_secure_daemon_handler \ - "${HADOOP_DAEMON_MODE}" \ - "${HADOOP_SUBCMD}" \ - "${HADOOP_CLASSNAME}" \ - "${daemon_pidfile}" \ - "${daemon_outfile}" \ - "${priv_pidfile}" \ - "${priv_outfile}" \ - "${priv_errfile}" \ - "${HADOOP_SUBCMD_ARGS[@]}" - else - hadoop_daemon_handler \ - "${HADOOP_DAEMON_MODE}" \ - "${HADOOP_SUBCMD}" \ - "${HADOOP_CLASSNAME}" \ - "${daemon_pidfile}" \ - "${daemon_outfile}" \ - "${HADOOP_SUBCMD_ARGS[@]}" - fi - exit $? -else - # shellcheck disable=SC2086 - hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}" -fi +# everything is in globals at this point, so call the generic handler +hadoop_generic_java_subcmd_handler \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh index cba37a44950..8c780632566 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh @@ -53,6 +53,9 @@ function hadoop_subproject_init hadoop_deprecate_envvar HADOOP_NFS3_SECURE_EXTRA_OPTS HDFS_NFS3_SECURE_EXTRA_OPTS + hadoop_deprecate_envvar HADOOP_SECURE_DN_USER HDFS_DATANODE_SECURE_USER + + hadoop_deprecate_envvar HADOOP_PRIVILEGED_NFS_USER HDFS_NFS3_SECURE_USER HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME}" @@ -74,6 +77,8 @@ if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hd_this}")" >/dev/null && pwd -P) fi +# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh + if [[ -n "${HADOOP_COMMON_HOME}" ]] && [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 06b33f977d0..6ff7e5aa9de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -142,7 +142,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { HdfsClientConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT; public static final String DFS_NAMENODE_HTTP_BIND_HOST_KEY = "dfs.namenode.http-bind-host"; - public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address"; + public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; public static final String DFS_NAMENODE_RPC_BIND_HOST_KEY = "dfs.namenode.rpc-bind-host"; public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY = "dfs.namenode.servicerpc-address"; public static final String DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY = "dfs.namenode.servicerpc-bind-host"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 23166e26f1b..47e1c0db306 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -449,19 +449,6 @@ public static Set getAllNnPrincipals(Configuration conf) throws IOExcept return principals; } - /** - * Returns list of InetSocketAddress corresponding to HA NN RPC addresses from - * the configuration. - * - * @param conf configuration - * @return list of InetSocketAddresses - */ - public static Map> getHaNnRpcAddresses( - Configuration conf) { - return DFSUtilClient.getAddresses(conf, null, - DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY); - } - /** * Returns list of InetSocketAddress corresponding to backup node rpc * addresses from the configuration. @@ -693,7 +680,7 @@ public static String addressMapToString( public static String nnAddressesAsString(Configuration conf) { Map> addresses = - getHaNnRpcAddresses(conf); + DFSUtilClient.getHaNnRpcAddresses(conf); return addressMapToString(addresses); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java index ea535e9a5f5..355608647c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java @@ -29,7 +29,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; -import static org.apache.hadoop.security.SecurityUtil.buildTokenService; import java.io.IOException; import java.net.InetSocketAddress; @@ -39,8 +38,6 @@ import java.util.List; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -48,17 +45,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo; import org.apache.hadoop.hdfs.protocol.ClientProtocol; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider; -import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -67,12 +59,6 @@ @InterfaceAudience.Private public class HAUtil { - private static final Log LOG = - LogFactory.getLog(HAUtil.class); - - private static final DelegationTokenSelector tokenSelector = - new DelegationTokenSelector(); - private static final String[] HA_SPECIAL_INDEPENDENT_KEYS = new String[]{ DFS_NAMENODE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_BIND_HOST_KEY, @@ -97,7 +83,7 @@ private HAUtil() { /* Hidden constructor */ } */ public static boolean isHAEnabled(Configuration conf, String nsId) { Map> addresses = - DFSUtil.getHaNnRpcAddresses(conf); + DFSUtilClient.getHaNnRpcAddresses(conf); if (addresses == null) return false; Map nnMap = addresses.get(nsId); return nnMap != null && nnMap.size() > 1; @@ -259,47 +245,6 @@ public static boolean useLogicalUri(Configuration conf, URI nameNodeUri) return provider.useLogicalURI(); } - /** - * Locate a delegation token associated with the given HA cluster URI, and if - * one is found, clone it to also represent the underlying namenode address. - * @param ugi the UGI to modify - * @param haUri the logical URI for the cluster - * @param nnAddrs collection of NNs in the cluster to which the token - * applies - */ - public static void cloneDelegationTokenForLogicalUri( - UserGroupInformation ugi, URI haUri, - Collection nnAddrs) { - // this cloning logic is only used by hdfs - Text haService = HAUtilClient.buildTokenServiceForLogicalUri(haUri, - HdfsConstants.HDFS_URI_SCHEME); - Token haToken = - tokenSelector.selectToken(haService, ugi.getTokens()); - if (haToken != null) { - for (InetSocketAddress singleNNAddr : nnAddrs) { - // this is a minor hack to prevent physical HA tokens from being - // exposed to the user via UGI.getCredentials(), otherwise these - // cloned tokens may be inadvertently propagated to jobs - Token specificToken = - haToken.privateClone(buildTokenService(singleNNAddr)); - Text alias = new Text( - HAUtilClient.buildTokenServicePrefixForLogicalUri( - HdfsConstants.HDFS_URI_SCHEME) - + "//" + specificToken.getService()); - ugi.addToken(alias, specificToken); - if (LOG.isDebugEnabled()) { - LOG.debug("Mapped HA service delegation token for logical URI " + - haUri + " to namenode " + singleNNAddr); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("No HA service delegation token found for logical URI " + - haUri); - } - } - } - /** * Get the internet address of the currently-active NN. This should rarely be * used, since callers of this method who connect directly to the NN using the diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index 61d701dfe88..d556c907c49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB; import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB; import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider; +import org.apache.hadoop.hdfs.server.namenode.ha.NameNodeHAProxyFactory; import org.apache.hadoop.hdfs.server.protocol.JournalProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.io.Text; @@ -112,7 +113,7 @@ public static ProxyAndInfo createProxy(Configuration conf, throws IOException { AbstractNNFailoverProxyProvider failoverProxyProvider = NameNodeProxiesClient.createFailoverProxyProvider(conf, nameNodeUri, - xface, true, fallbackToSimpleAuth); + xface, true, fallbackToSimpleAuth, new NameNodeHAProxyFactory()); if (failoverProxyProvider == null) { return createNonHAProxy(conf, DFSUtilClient.getNNAddress(nameNodeUri), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 1944fe7a287..ab0ccdb1ea9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -424,7 +424,8 @@ public CreateResponseProto create(RpcController controller, PBHelperClient.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(), (short) req.getReplication(), req.getBlockSize(), PBHelperClient.convertCryptoProtocolVersions( - req.getCryptoProtocolVersionList())); + req.getCryptoProtocolVersionList()), + req.getEcPolicyName()); if (result != null) { return CreateResponseProto.newBuilder().setFs(PBHelperClient.convert(result)) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index 4493772d711..414d3a7933e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIOException; @@ -275,12 +276,10 @@ public static class StorageDirectory implements FormatConfirmable { private final StorageLocation location; public StorageDirectory(File dir) { - // default dirType is null this(dir, null, false); } public StorageDirectory(StorageLocation location) { - // default dirType is null this(null, false, location); } @@ -337,7 +336,8 @@ private StorageDirectory(File dir, StorageDirType dirType, boolean isShared, StorageLocation location) { this.root = dir; this.lock = null; - this.dirType = dirType; + // default dirType is UNDEFINED + this.dirType = (dirType == null ? NameNodeDirType.UNDEFINED : dirType); this.isShared = isShared; this.location = location; assert location == null || dir == null || diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 00e6b3ecb01..e0daca78b9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -70,7 +70,8 @@ class BPOfferService { * handshake. */ volatile DatanodeRegistration bpRegistration; - + + private final String nameserviceId; private final DataNode dn; /** @@ -120,12 +121,16 @@ void writeUnlock() { mWriteLock.unlock(); } - BPOfferService(List nnAddrs, - List lifelineNnAddrs, DataNode dn) { + BPOfferService( + final String nameserviceId, + List nnAddrs, + List lifelineNnAddrs, + DataNode dn) { Preconditions.checkArgument(!nnAddrs.isEmpty(), "Must pass at least one NN."); Preconditions.checkArgument(nnAddrs.size() == lifelineNnAddrs.size(), "Must pass same number of NN addresses and lifeline addresses."); + this.nameserviceId = nameserviceId; this.dn = dn; for (int i = 0; i < nnAddrs.size(); ++i) { @@ -170,6 +175,14 @@ boolean isAlive() { return false; } + /** + * Gets nameservice id to which this {@link BPOfferService} maps to. + * @return nameservice id, which can be null. + */ + String getNameserviceId() { + return nameserviceId; + } + String getBlockPoolId() { readLock(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index ddc28b72a7a..21e2a3b518e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -25,7 +25,6 @@ import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -279,7 +278,10 @@ private void connectToNNAndHandshake() throws IOException { // This also initializes our block pool in the DN if we are // the first NN connection for this BP. bpos.verifyAndSetNamespaceInfo(this, nsInfo); - + + /* set thread name again to include NamespaceInfo when it's available. */ + this.bpThread.setName(formatThreadName("heartbeating", nnAddr)); + // Second phase of the handshake with the NN. register(nsInfo); } @@ -547,14 +549,15 @@ void start() { lifelineSender.start(); } } - - private String formatThreadName(String action, InetSocketAddress addr) { - Collection dataDirs = - DataNode.getStorageLocations(dn.getConf()); - return "DataNode: [" + dataDirs.toString() + "] " + - action + " to " + addr; + + private String formatThreadName( + final String action, + final InetSocketAddress addr) { + final String prefix = bpos.getBlockPoolId() != null ? bpos.getBlockPoolId() + : bpos.getNameserviceId(); + return prefix + " " + action + " to " + addr; } - + //This must be called only by blockPoolManager. void stop() { shouldServiceRun = false; @@ -1008,8 +1011,8 @@ public void run() { } public void start() { - lifelineThread = new Thread(this, formatThreadName("lifeline", - lifelineNnAddr)); + lifelineThread = new Thread(this, + formatThreadName("lifeline", lifelineNnAddr)); lifelineThread.setDaemon(true); lifelineThread.setUncaughtExceptionHandler( new Thread.UncaughtExceptionHandler() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java index 497f4eddddd..75f59df2a47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java @@ -225,7 +225,7 @@ private void doRefreshNamenodes( lifelineAddrs.add(nnIdToLifelineAddr != null ? nnIdToLifelineAddr.get(nnId) : null); } - BPOfferService bpos = createBPOS(addrs, lifelineAddrs); + BPOfferService bpos = createBPOS(nsToAdd, addrs, lifelineAddrs); bpByNameserviceId.put(nsToAdd, bpos); offerServices.add(bpos); } @@ -275,8 +275,10 @@ private void doRefreshNamenodes( /** * Extracted out for test purposes. */ - protected BPOfferService createBPOS(List nnAddrs, + protected BPOfferService createBPOS( + final String nameserviceId, + List nnAddrs, List lifelineNnAddrs) { - return new BPOfferService(nnAddrs, lifelineNnAddrs, dn); + return new BPOfferService(nameserviceId, nnAddrs, lifelineNnAddrs, dn); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index f838fd913e0..706d93a6f0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -241,12 +241,12 @@ public void run() { LOG.info("Failed to read expected encryption handshake from client " + "at " + peer.getRemoteAddressString() + ". Perhaps the client " + "is running an older version of Hadoop which does not support " + - "encryption"); + "encryption", imne); } else { LOG.info("Failed to read expected SASL data transfer protection " + "handshake from client at " + peer.getRemoteAddressString() + ". Perhaps the client is running an older version of Hadoop " + - "which does not support SASL data transfer protection"); + "which does not support SASL data transfer protection", imne); } return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java index 7584d97b5f2..b71c0156f03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java @@ -187,28 +187,21 @@ public void onFailure(@Nonnull Throwable t) { /** * {@inheritDoc}. + * + * The results of in-progress checks are not useful during shutdown, + * so we optimize for faster shutdown by interrupt all actively + * executing checks. */ @Override public void shutdownAndWait(long timeout, TimeUnit timeUnit) throws InterruptedException { - // Try orderly shutdown. - executorService.shutdown(); - - if (!executorService.awaitTermination(timeout, timeUnit)) { - // Interrupt executing tasks and wait again. - executorService.shutdownNow(); - executorService.awaitTermination(timeout, timeUnit); - } if (scheduledExecutorService != null) { - // Try orderly shutdown - scheduledExecutorService.shutdown(); - - if (!scheduledExecutorService.awaitTermination(timeout, timeUnit)) { - // Interrupt executing tasks and wait again. - scheduledExecutorService.shutdownNow(); - scheduledExecutorService.awaitTermination(timeout, timeUnit); - } + scheduledExecutorService.shutdownNow(); + scheduledExecutorService.awaitTermination(timeout, timeUnit); } + + executorService.shutdownNow(); + executorService.awaitTermination(timeout, timeUnit); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index bc75f0faa01..7eac87d92eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -375,10 +375,15 @@ private void processRecursively(String parent, HdfsFileStatus status, /** @return true if it is necessary to run another round of migration */ private void processFile(String fullPath, HdfsLocatedFileStatus status, Result result) { - final byte policyId = status.getStoragePolicy(); - // currently we ignore files with unspecified storage policy + byte policyId = status.getStoragePolicy(); if (policyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) { - return; + try { + // get default policy from namenode + policyId = dfs.getServerDefaults().getDefaultStoragePolicyId(); + } catch (IOException e) { + LOG.warn("Failed to get default policy for " + fullPath, e); + return; + } } final BlockStoragePolicy policy = blockStoragePolicies[policyId]; if (policy == null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java index 8edaed69033..2fb369c9504 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java @@ -17,23 +17,15 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.io.IOException; -import java.net.InetSocketAddress; - -import javax.servlet.ServletContext; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.NameNodeProxies; -import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.server.common.JspHelper; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.UserGroupInformation; -import org.znerd.xmlenc.XMLOutputter; /** * A base class for the servlets in DFS. @@ -44,44 +36,6 @@ abstract class DfsServlet extends HttpServlet { static final Log LOG = LogFactory.getLog(DfsServlet.class.getCanonicalName()); - /** Write the object to XML format */ - protected void writeXml(Exception except, String path, XMLOutputter doc) - throws IOException { - doc.startTag(RemoteException.class.getSimpleName()); - doc.attribute("path", path); - if (except instanceof RemoteException) { - doc.attribute("class", ((RemoteException) except).getClassName()); - } else { - doc.attribute("class", except.getClass().getName()); - } - String msg = except.getLocalizedMessage(); - int i = msg.indexOf("\n"); - if (i >= 0) { - msg = msg.substring(0, i); - } - doc.attribute("message", msg.substring(msg.indexOf(":") + 1).trim()); - doc.endTag(); - } - - /** - * Create a {@link NameNode} proxy from the current {@link ServletContext}. - */ - protected ClientProtocol createNameNodeProxy() throws IOException { - ServletContext context = getServletContext(); - // if we are running in the Name Node, use it directly rather than via - // rpc - NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context); - if (nn != null) { - return nn.getRpcServer(); - } - InetSocketAddress nnAddr = - NameNodeHttpServer.getNameNodeAddressFromContext(context); - Configuration conf = new HdfsConfiguration( - NameNodeHttpServer.getConfFromContext(context)); - return NameNodeProxies.createProxy(conf, DFSUtilClient.getNNUri(nnAddr), - ClientProtocol.class).getProxy(); - } - protected UserGroupInformation getUGI(HttpServletRequest request, Configuration conf) throws IOException { return JspHelper.getUGI(getServletContext(), request, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index c23b034ac90..177c0e09fe5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -20,11 +20,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; -import java.util.Arrays; import java.util.Map; import java.util.TreeMap; import java.util.stream.Collectors; @@ -40,50 +39,12 @@ @InterfaceAudience.LimitedPrivate({"HDFS"}) public final class ErasureCodingPolicyManager { - /** - * TODO: HDFS-8095. - */ - private static final int DEFAULT_CELLSIZE = 64 * 1024; - private static final ErasureCodingPolicy SYS_POLICY1 = - new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_SCHEMA, - DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_POLICY_ID); - private static final ErasureCodingPolicy SYS_POLICY2 = - new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, - DEFAULT_CELLSIZE, HdfsConstants.RS_3_2_POLICY_ID); - private static final ErasureCodingPolicy SYS_POLICY3 = - new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA, - DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_LEGACY_POLICY_ID); - private static final ErasureCodingPolicy SYS_POLICY4 = - new ErasureCodingPolicy(ErasureCodeConstants.XOR_2_1_SCHEMA, - DEFAULT_CELLSIZE, HdfsConstants.XOR_2_1_POLICY_ID); - private static final ErasureCodingPolicy SYS_POLICY5 = - new ErasureCodingPolicy(ErasureCodeConstants.RS_10_4_SCHEMA, - DEFAULT_CELLSIZE, HdfsConstants.RS_10_4_POLICY_ID); - - //We may add more later. - private static final ErasureCodingPolicy[] SYS_POLICIES = - new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3, - SYS_POLICY4, SYS_POLICY5}; - // Supported storage policies for striped EC files private static final byte[] SUITABLE_STORAGE_POLICIES_FOR_EC_STRIPED_MODE = new byte[]{ HdfsConstants.HOT_STORAGE_POLICY_ID, HdfsConstants.COLD_STORAGE_POLICY_ID, HdfsConstants.ALLSSD_STORAGE_POLICY_ID}; - /** - * All supported policies maintained in NN memory for fast querying, - * identified and sorted by its name. - */ - private static final Map SYSTEM_POLICIES_BY_NAME; - - static { - // Create a hashmap of all available policies for quick lookup by name - SYSTEM_POLICIES_BY_NAME = new TreeMap<>(); - for (ErasureCodingPolicy policy : SYS_POLICIES) { - SYSTEM_POLICIES_BY_NAME.put(policy.getName(), policy); - } - } /** * All enabled policies maintained in NN memory for fast querying, @@ -101,9 +62,10 @@ public final class ErasureCodingPolicyManager { if (policyName.trim().isEmpty()) { continue; } - ErasureCodingPolicy ecPolicy = SYSTEM_POLICIES_BY_NAME.get(policyName); + ErasureCodingPolicy ecPolicy = + SystemErasureCodingPolicies.getByName(policyName); if (ecPolicy == null) { - String sysPolicies = Arrays.asList(SYS_POLICIES).stream() + String sysPolicies = SystemErasureCodingPolicies.getPolicies().stream() .map(ErasureCodingPolicy::getName) .collect(Collectors.joining(", ")); String msg = String.format("EC policy '%s' specified at %s is not a " + @@ -124,35 +86,6 @@ public final class ErasureCodingPolicyManager { */ } - /** - * Get system defined policies. - * @return system policies - */ - public static ErasureCodingPolicy[] getSystemPolicies() { - return SYS_POLICIES; - } - - /** - * Get a policy by policy ID. - * @return ecPolicy, or null if not found - */ - public static ErasureCodingPolicy getPolicyByID(byte id) { - for (ErasureCodingPolicy policy : SYS_POLICIES) { - if (policy.getId() == id) { - return policy; - } - } - return null; - } - - /** - * Get a policy by policy name. - * @return ecPolicy, or null if not found - */ - public static ErasureCodingPolicy getPolicyByName(String name) { - return SYSTEM_POLICIES_BY_NAME.get(name); - } - /** * Get the set of enabled policies. * @return all policies @@ -190,6 +123,7 @@ public static boolean checkStoragePolicySuitableForECStripedMode( * Clear and clean up. */ public void clear() { - enabledPoliciesByName.clear(); + // TODO: we should only clear policies loaded from NN metadata. + // This is a placeholder for HDFS-7337. } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 50843c87066..763b935b303 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.XAttrHelper; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; @@ -57,6 +58,39 @@ final class FSDirErasureCodingOp { */ private FSDirErasureCodingOp() {} + /** + * Check if the ecPolicyName is valid and enabled, return the corresponding + * EC policy if is. + * @param fsn namespace + * @param ecPolicyName name of EC policy to be checked + * @return an erasure coding policy if ecPolicyName is valid and enabled + * @throws IOException + */ + static ErasureCodingPolicy getErasureCodingPolicyByName( + final FSNamesystem fsn, final String ecPolicyName) throws IOException { + assert fsn.hasReadLock(); + ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager() + .getEnabledPolicyByName(ecPolicyName); + if (ecPolicy == null) { + final String sysPolicies = + Arrays.asList( + fsn.getErasureCodingPolicyManager().getEnabledPolicies()) + .stream() + .map(ErasureCodingPolicy::getName) + .collect(Collectors.joining(", ")); + final String message = String.format("Policy '%s' does not match any " + + "enabled erasure" + + " coding policies: [%s]. The set of enabled erasure coding " + + "policies can be configured at '%s'.", + ecPolicyName, + sysPolicies, + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY + ); + throw new HadoopIllegalArgumentException(message); + } + return ecPolicy; + } + /** * Set an erasure coding policy on the given path. * @@ -83,25 +117,8 @@ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn, List xAttrs; fsd.writeLock(); try { - ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager() - .getEnabledPolicyByName(ecPolicyName); - if (ecPolicy == null) { - final String sysPolicies = - Arrays.asList( - fsn.getErasureCodingPolicyManager().getEnabledPolicies()) - .stream() - .map(ErasureCodingPolicy::getName) - .collect(Collectors.joining(", ")); - final String message = String.format("Policy '%s' does not match any " + - "enabled erasure" + - " coding policies: [%s]. The set of enabled erasure coding " + - "policies can be configured at '%s'.", - ecPolicyName, - sysPolicies, - DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY - ); - throw new HadoopIllegalArgumentException(message); - } + ErasureCodingPolicy ecPolicy = getErasureCodingPolicyByName(fsn, + ecPolicyName); iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); // Write access is required to set erasure coding policy if (fsd.isPermissionEnabled()) { @@ -302,7 +319,7 @@ private static ErasureCodingPolicy getErasureCodingPolicyForPath( if (inode.isFile()) { byte id = inode.asFile().getErasureCodingPolicyID(); return id < 0 ? null : - ErasureCodingPolicyManager.getPolicyByID(id); + SystemErasureCodingPolicies.getByID(id); } // We don't allow setting EC policies on paths with a symlink. Thus // if a symlink is encountered, the dir shouldn't have EC policy. @@ -317,8 +334,7 @@ private static ErasureCodingPolicy getErasureCodingPolicyForPath( ByteArrayInputStream bIn = new ByteArrayInputStream(xattr.getValue()); DataInputStream dIn = new DataInputStream(bIn); String ecPolicyName = WritableUtils.readString(dIn); - return ErasureCodingPolicyManager - .getPolicyByName(ecPolicyName); + return SystemErasureCodingPolicies.getByName(ecPolicyName); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index bb920049b33..7bf291634db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.hdfs.AddBlockFlag; import org.apache.hadoop.fs.CreateFlag; @@ -351,7 +352,7 @@ static HdfsFileStatus startFile( EnumSet flag, boolean createParent, short replication, long blockSize, FileEncryptionInfo feInfo, INode.BlocksMapUpdateInfo toRemoveBlocks, - boolean logRetryEntry) + String ecPolicyName, boolean logRetryEntry) throws IOException { assert fsn.hasWriteLock(); boolean overwrite = flag.contains(CreateFlag.OVERWRITE); @@ -385,7 +386,7 @@ static HdfsFileStatus startFile( FSDirMkdirOp.createAncestorDirectories(fsd, iip, permissions); if (parent != null) { iip = addFile(fsd, parent, iip.getLastLocalName(), permissions, - replication, blockSize, holder, clientMachine); + replication, blockSize, holder, clientMachine, ecPolicyName); newNode = iip != null ? iip.getLastINode().asFile() : null; } if (newNode == null) { @@ -521,7 +522,7 @@ private static BlockInfo addBlock(FSDirectory fsd, String path, private static INodesInPath addFile( FSDirectory fsd, INodesInPath existing, byte[] localName, PermissionStatus permissions, short replication, long preferredBlockSize, - String clientName, String clientMachine) + String clientName, String clientMachine, String ecPolicyName) throws IOException { Preconditions.checkNotNull(existing); @@ -530,8 +531,14 @@ private static INodesInPath addFile( fsd.writeLock(); try { boolean isStriped = false; - ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp. - unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing); + ErasureCodingPolicy ecPolicy; + if (!StringUtils.isEmpty(ecPolicyName)) { + ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName( + fsd.getFSNamesystem(), ecPolicyName); + } else { + ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsd.getFSNamesystem(), existing); + } if (ecPolicy != null) { isStriped = true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index eab728283b3..9c89be15484 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; @@ -335,7 +336,7 @@ private INodeFile loadINodeFile(INodeSection.INode n) { assert ((!isStriped) || (isStriped && !f.hasReplication())); Short replication = (!isStriped ? (short) f.getReplication() : null); ErasureCodingPolicy ecPolicy = isStriped ? - ErasureCodingPolicyManager.getPolicyByID( + SystemErasureCodingPolicies.getByID( (byte) f.getErasureCodingPolicyID()) : null; Byte ecPolicyID = (isStriped ? ecPolicy.getId() : null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index f405cc0d70a..e24778f9c9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -148,6 +148,7 @@ import org.apache.hadoop.hdfs.AddBlockFlag; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileEncryptionInfo; @@ -778,8 +779,12 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), conf.getBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, DFS_ENCRYPT_DATA_TRANSFER_DEFAULT), conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT), - checksumType); - + checksumType, + conf.getTrimmed( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + ""), + blockManager.getStoragePolicySuite().getDefaultPolicy().getId()); + this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, DFS_NAMENODE_MAX_OBJECTS_DEFAULT); @@ -2175,14 +2180,14 @@ CryptoProtocolVersion chooseProtocolVersion( */ HdfsFileStatus startFile(String src, PermissionStatus permissions, String holder, String clientMachine, EnumSet flag, - boolean createParent, short replication, long blockSize, - CryptoProtocolVersion[] supportedVersions, boolean logRetryCache) - throws IOException { + boolean createParent, short replication, long blockSize, + CryptoProtocolVersion[] supportedVersions, String ecPolicyName, + boolean logRetryCache) throws IOException { HdfsFileStatus status; try { status = startFileInt(src, permissions, holder, clientMachine, flag, - createParent, replication, blockSize, supportedVersions, + createParent, replication, blockSize, supportedVersions, ecPolicyName, logRetryCache); } catch (AccessControlException e) { logAuditEvent(false, "create", src); @@ -2196,8 +2201,7 @@ private HdfsFileStatus startFileInt(String src, PermissionStatus permissions, String holder, String clientMachine, EnumSet flag, boolean createParent, short replication, long blockSize, CryptoProtocolVersion[] supportedVersions, - boolean logRetryCache) - throws IOException { + String ecPolicyName, boolean logRetryCache) throws IOException { if (NameNode.stateChangeLog.isDebugEnabled()) { StringBuilder builder = new StringBuilder(); builder.append("DIR* NameSystem.startFile: src=").append(src) @@ -2265,9 +2269,8 @@ private HdfsFileStatus startFileInt(String src, dir.writeLock(); try { stat = FSDirWriteFileOp.startFile(this, iip, permissions, holder, - clientMachine, flag, createParent, - replication, blockSize, feInfo, - toRemoveBlocks, logRetryCache); + clientMachine, flag, createParent, replication, blockSize, feInfo, + toRemoveBlocks, ecPolicyName, logRetryCache); } catch (IOException e) { skipSync = e instanceof StandbyException; throw e; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 3da6aa7c816..f35bf3ce5ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; @@ -191,8 +192,8 @@ static long getBlockLayoutRedundancy(final BlockType blockType, if (blockType == STRIPED) { Preconditions.checkArgument(replication == null && erasureCodingPolicyID != null); - Preconditions.checkArgument(ErasureCodingPolicyManager - .getPolicyByID(erasureCodingPolicyID) != null, + Preconditions.checkArgument(SystemErasureCodingPolicies + .getByID(erasureCodingPolicyID) != null, "Could not find EC policy with ID 0x" + StringUtils .byteToHexString(erasureCodingPolicyID)); layoutRedundancy |= BLOCK_TYPE_MASK_STRIPED; @@ -516,8 +517,7 @@ public short getPreferredBlockReplication() { } ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getPolicyByID( - getErasureCodingPolicyID()); + SystemErasureCodingPolicies.getByID(getErasureCodingPolicyID()); Preconditions.checkNotNull(ecPolicy, "Could not find EC policy with ID 0x" + StringUtils.byteToHexString(getErasureCodingPolicyID())); return (short) (ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java index b37321d6243..1d5dbf674a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java @@ -238,7 +238,7 @@ public static INodesInPath append(INodesInPath iip, INode child, } private final byte[][] path; - private final String pathname; + private volatile String pathname; /** * Array with the specified number of INodes resolved for a given path. @@ -268,7 +268,6 @@ private INodesInPath(INode[] inodes, byte[][] path, boolean isRaw, Preconditions.checkArgument(inodes != null && path != null); this.inodes = inodes; this.path = path; - this.pathname = DFSUtil.byteArray2PathString(path); this.isRaw = isRaw; this.isSnapshot = isSnapshot; this.snapshotId = snapshotId; @@ -329,6 +328,9 @@ public byte[] getPathComponent(int i) { /** @return the full path in string form */ public String getPath() { + if (pathname == null) { + pathname = DFSUtil.byteArray2PathString(path); + } return pathname; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index f792e8a9a47..e477b81062a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -722,8 +722,8 @@ public FsServerDefaults getServerDefaults() throws IOException { @Override // ClientProtocol public HdfsFileStatus create(String src, FsPermission masked, String clientName, EnumSetWritable flag, - boolean createParent, short replication, long blockSize, - CryptoProtocolVersion[] supportedVersions) + boolean createParent, short replication, long blockSize, + CryptoProtocolVersion[] supportedVersions, String ecPolicyName) throws IOException { checkNNStartup(); String clientMachine = getClientMachine(); @@ -747,7 +747,7 @@ public HdfsFileStatus create(String src, FsPermission masked, .getShortUserName(), null, masked); status = namesystem.startFile(src, perm, clientName, clientMachine, flag.get(), createParent, replication, blockSize, supportedVersions, - cacheEntry != null); + ecPolicyName, cacheEntry != null); } finally { RetryCache.setState(cacheEntry, status != null, status); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/NameNodeHAProxyFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/NameNodeHAProxyFactory.java new file mode 100644 index 00000000000..036b6eb367d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/NameNodeHAProxyFactory.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.ha; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.NameNodeProxies; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.AtomicBoolean; + +public class NameNodeHAProxyFactory implements HAProxyFactory { + + @Override + public T createProxy(Configuration conf, InetSocketAddress nnAddr, + Class xface, UserGroupInformation ugi, boolean withRetries, + AtomicBoolean fallbackToSimpleAuth) throws IOException { + return NameNodeProxies.createNonHAProxy(conf, nnAddr, xface, + ugi, withRetries, fallbackToSimpleAuth).getProxy(); + } + + @Override + public T createProxy(Configuration conf, InetSocketAddress nnAddr, + Class xface, UserGroupInformation ugi, boolean withRetries) + throws IOException { + return NameNodeProxies.createNonHAProxy(conf, nnAddr, xface, + ugi, withRetries).getProxy(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/robots.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/robots.txt new file mode 100644 index 00000000000..1f53798bb4f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/robots.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/robots.txt new file mode 100644 index 00000000000..1f53798bb4f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/robots.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/robots.txt new file mode 100644 index 00000000000..1f53798bb4f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md index 56a2ab8c15e..91ad107da5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md @@ -98,7 +98,7 @@ The effective storage policy can be retrieved by the "[`storagepolicies -getStor Mover - A New Data Migration Tool --------------------------------- -A new data migration tool is added for archiving data. The tool is similar to Balancer. It periodically scans the files in HDFS to check if the block placement satisfies the storage policy. For the blocks violating the storage policy, it moves the replicas to a different storage type in order to fulfill the storage policy requirement. +A new data migration tool is added for archiving data. The tool is similar to Balancer. It periodically scans the files in HDFS to check if the block placement satisfies the storage policy. For the blocks violating the storage policy, it moves the replicas to a different storage type in order to fulfill the storage policy requirement. Note that it always tries to move block replicas within the same node whenever possible. If that is not possible (e.g. when a node doesn’t have the target storage type) then it will copy the block replicas to another node over the network. * Command: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md index f150b8b25fc..bd87b909975 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md @@ -89,10 +89,10 @@ The rest of the NFS gateway configurations are optional for both secure and non- server can be assured to have been committed. * HDFS super-user is the user with the same identity as NameNode process itself and - the super-user can do anything in that permissions checks never fail for the super-user. + the super-user can do anything in that permissions checks never fail for the super-user. If the following property is configured, the superuser on NFS client can access any file on HDFS. By default, the super user is not configured in the gateway. - Note that, even the the superuser is configured, "nfs.exports.allowed.hosts" still takes effect. + Note that, even the the superuser is configured, "nfs.exports.allowed.hosts" still takes effect. For example, the superuser will not have write access to HDFS files through the gateway if the NFS client host is not allowed to have write access in "nfs.exports.allowed.hosts". @@ -143,7 +143,7 @@ It's strongly recommended for the users to update a few configuration properties For example: "192.168.0.0/22 rw ; \\\\w\*\\\\.example\\\\.com ; host1.test.org ro;". Only the NFS gateway needs to restart after this property is updated. Note that, here Java regular expression is different with the regulation expression used in Linux NFS export table, such as, using "\\\\w\*\\\\.example\\\\.com" instead of "\*.example.com", "192\\\\.168\\\\.0\\\\.(11|22)" - instead of "192.168.0.[11|22]" and so on. + instead of "192.168.0.[11|22]" and so on. nfs.exports.allowed.hosts @@ -151,10 +151,10 @@ It's strongly recommended for the users to update a few configuration properties * HDFS super-user is the user with the same identity as NameNode process itself and - the super-user can do anything in that permissions checks never fail for the super-user. + the super-user can do anything in that permissions checks never fail for the super-user. If the following property is configured, the superuser on NFS client can access any file on HDFS. By default, the super user is not configured in the gateway. - Note that, even the the superuser is configured, "nfs.exports.allowed.hosts" still takes effect. + Note that, even the the superuser is configured, "nfs.exports.allowed.hosts" still takes effect. For example, the superuser will not have write access to HDFS files through the gateway if the NFS client host is not allowed to have write access in "nfs.exports.allowed.hosts". @@ -224,7 +224,7 @@ Three daemons are required to provide NFS service: rpcbind (or portmap), mountd [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop nfs3 [root]> $HADOOP_HOME/bin/hdfs --daemon stop portmap -Optionally, you can forgo running the Hadoop-provided portmap daemon and instead use the system portmap daemon on all operating systems if you start the NFS Gateway as root. This will allow the HDFS NFS Gateway to work around the aforementioned bug and still register using the system portmap daemon. To do so, just start the NFS gateway daemon as you normally would, but make sure to do so as the "root" user, and also set the "HADOOP\_PRIVILEGED\_NFS\_USER" environment variable to an unprivileged user. In this mode the NFS Gateway will start as root to perform its initial registration with the system portmap, and then will drop privileges back to the user specified by the HADOOP\_PRIVILEGED\_NFS\_USER afterward and for the rest of the duration of the lifetime of the NFS Gateway process. Note that if you choose this route, you should skip steps 1 and 2 above. +Optionally, you can forgo running the Hadoop-provided portmap daemon and instead use the system portmap daemon on all operating systems if you start the NFS Gateway as root. This will allow the HDFS NFS Gateway to work around the aforementioned bug and still register using the system portmap daemon. To do so, just start the NFS gateway daemon as you normally would, but make sure to do so as the "root" user, and also set the "HDFS\_NFS3\_SECURE\_USER" environment variable to an unprivileged user. In this mode the NFS Gateway will start as root to perform its initial registration with the system portmap, and then will drop privileges back to the user specified by the HDFS\_NFS3\_SECURE\_USER afterward and for the rest of the duration of the lifetime of the NFS Gateway process. Note that if you choose this route, you should skip steps 1 and 2 above. Verify validity of NFS related services --------------------------------------- @@ -268,7 +268,7 @@ Verify validity of NFS related services Mount the export "/" -------------------- -Currently NFS v3 only uses TCP as the transportation protocol. NLM is not supported so mount option "nolock" is needed. +Currently NFS v3 only uses TCP as the transportation protocol. NLM is not supported so mount option "nolock" is needed. Mount option "sync" is strongly recommended since it can minimize or avoid reordered writes, which results in more predictable throughput. Not specifying the sync option may cause unreliable behavior when uploading large files. It's recommended to use hard mount. This is because, even after the client sends all data to NFS gateway, it may take NFS gateway some extra time to transfer data to HDFS when writes were reorderd by NFS client Kernel. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md index 089d7b47011..3f9fbf07d1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md @@ -97,6 +97,7 @@ Once a KMS has been set up and the NameNode and HDFS clients have been correctly #### hadoop.security.key.provider.path The KeyProvider to use when interacting with encryption keys used when reading and writing to an encryption zone. +HDFS clients will use the provider path returned from Namenode via getServerDefaults. If namenode doesn't support returning key provider uri then client's conf will be used. ### Selecting an encryption algorithm and codec diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 9b782f32f2a..9dccad5a96c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -113,6 +113,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -138,7 +139,6 @@ import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -283,8 +283,7 @@ public static void setEditLogForTesting(FSNamesystem fsn, FSEditLog newLog) { public static void enableAllECPolicies(Configuration conf) { // Enable all the available EC policies - String policies = - Arrays.asList(ErasureCodingPolicyManager.getSystemPolicies()).stream() + String policies = SystemErasureCodingPolicies.getPolicies().stream() .map(ErasureCodingPolicy::getName) .collect(Collectors.joining(",")); conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, policies); @@ -1967,7 +1966,7 @@ public static void createStripedFile(MiniDFSCluster cluster, Path file, .create(file.toString(), new FsPermission((short)0755), dfs.getClient().getClientName(), new EnumSetWritable<>(EnumSet.of(CreateFlag.CREATE)), - false, (short)1, 128*1024*1024L, null); + false, (short)1, 128*1024*1024L, null, null); FSNamesystem ns = cluster.getNamesystem(); FSDirectory fsdir = ns.getFSDirectory(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java index 8008ed3396e..1bab5dbbd9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java @@ -27,13 +27,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.impl.BlockReaderTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem.WebHdfsInputStream; import org.apache.hadoop.io.IOUtils; @@ -567,7 +566,6 @@ public static LocatedBlocks getLocatedBlocks(Path file, * @return ErasureCodingPolicy */ public static ErasureCodingPolicy getDefaultECPolicy() { - return ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.RS_6_3_POLICY_ID); + return SystemErasureCodingPolicies.getPolicies().get(0); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java index 7e8621b329c..6265f44aa20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java @@ -42,10 +42,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.IPFailoverProxyProvider; +import org.apache.hadoop.hdfs.server.namenode.ha.HAProxyFactory; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.retry.FailoverProxyProvider; @@ -333,7 +333,7 @@ public static class DummyLegacyFailoverProxyProvider private Class xface; private T proxy; public DummyLegacyFailoverProxyProvider(Configuration conf, URI uri, - Class xface) { + Class xface, HAProxyFactory proxyFactory) { try { this.proxy = NameNodeProxies.createNonHAProxy(conf, DFSUtilClient.getNNAddress(uri), xface, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 6db70d59d57..7a71df8d90b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -269,7 +269,8 @@ public Object answer(InvocationOnMock invocation) .when(mockNN) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable) anyObject(), anyBoolean(), - anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject()); + anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(), + anyObject()); final DFSClient client = new DFSClient(null, mockNN, conf, null); OutputStream os = client.create("testfile", true); @@ -370,7 +371,7 @@ public void testLeaseRenewSocketTimeout() throws Exception String file1 = "/testFile1"; String file2 = "/testFile2"; // Set short retry timeouts so this test runs faster - conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10); + conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10); conf.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, 2 * 1000); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java index 9ec01b6dbc2..52e3bb4f58d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hdfs; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.EnumSet; @@ -41,10 +43,13 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; +import org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; import org.apache.htrace.core.SpanId; import org.junit.AfterClass; import org.junit.Assert; @@ -64,6 +69,9 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; + public class TestDFSOutputStream { static MiniDFSCluster cluster; @@ -133,6 +141,124 @@ public void testComputePacketChunkSize() throws Exception { Assert.assertTrue((Integer) field.get(dos) + 257 < packetSize); } + /** + * This tests preventing overflows of package size and bodySize. + *

+ * See also https://issues.apache.org/jira/browse/HDFS-11608. + *

+ * @throws IOException + * @throws SecurityException + * @throws NoSuchFieldException + * @throws InvocationTargetException + * @throws IllegalArgumentException + * @throws IllegalAccessException + * @throws NoSuchMethodException + */ + @Test(timeout=60000) + public void testPreventOverflow() throws IOException, NoSuchFieldException, + SecurityException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException { + + final int defaultWritePacketSize = DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; + int configuredWritePacketSize = defaultWritePacketSize; + int finalWritePacketSize = defaultWritePacketSize; + + /* test default WritePacketSize, e.g. 64*1024 */ + runAdjustChunkBoundary(configuredWritePacketSize, finalWritePacketSize); + + /* test large WritePacketSize, e.g. 1G */ + configuredWritePacketSize = 1000 * 1024 * 1024; + finalWritePacketSize = PacketReceiver.MAX_PACKET_SIZE; + runAdjustChunkBoundary(configuredWritePacketSize, finalWritePacketSize); + } + + /** + * @configuredWritePacketSize the configured WritePacketSize. + * @finalWritePacketSize the final WritePacketSize picked by + * {@link DFSOutputStream#adjustChunkBoundary} + */ + private void runAdjustChunkBoundary( + final int configuredWritePacketSize, + final int finalWritePacketSize) throws IOException, NoSuchFieldException, + SecurityException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException { + + final boolean appendChunk = false; + final long blockSize = 3221225500L; + final long bytesCurBlock = 1073741824L; + final int bytesPerChecksum = 512; + final int checksumSize = 4; + final int chunkSize = bytesPerChecksum + checksumSize; + final int packateMaxHeaderLength = 33; + + MiniDFSCluster dfsCluster = null; + final File baseDir = new File(PathUtils.getTestDir(getClass()), + GenericTestUtils.getMethodName()); + + try { + final Configuration dfsConf = new Configuration(); + dfsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, + baseDir.getAbsolutePath()); + dfsConf.setInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, + configuredWritePacketSize); + dfsCluster = new MiniDFSCluster.Builder(dfsConf).numDataNodes(1).build(); + dfsCluster.waitActive(); + + final FSDataOutputStream os = dfsCluster.getFileSystem() + .create(new Path(baseDir.getAbsolutePath(), "testPreventOverflow")); + final DFSOutputStream dos = (DFSOutputStream) Whitebox + .getInternalState(os, "wrappedStream"); + + /* set appendChunk */ + final Method setAppendChunkMethod = dos.getClass() + .getDeclaredMethod("setAppendChunk", boolean.class); + setAppendChunkMethod.setAccessible(true); + setAppendChunkMethod.invoke(dos, appendChunk); + + /* set bytesCurBlock */ + final Method setBytesCurBlockMethod = dos.getClass() + .getDeclaredMethod("setBytesCurBlock", long.class); + setBytesCurBlockMethod.setAccessible(true); + setBytesCurBlockMethod.invoke(dos, bytesCurBlock); + + /* set blockSize */ + final Field blockSizeField = dos.getClass().getDeclaredField("blockSize"); + blockSizeField.setAccessible(true); + blockSizeField.setLong(dos, blockSize); + + /* call adjustChunkBoundary */ + final Method method = dos.getClass() + .getDeclaredMethod("adjustChunkBoundary"); + method.setAccessible(true); + method.invoke(dos); + + /* get and verify writePacketSize */ + final Field writePacketSizeField = dos.getClass() + .getDeclaredField("writePacketSize"); + writePacketSizeField.setAccessible(true); + Assert.assertEquals(writePacketSizeField.getInt(dos), + finalWritePacketSize); + + /* get and verify chunksPerPacket */ + final Field chunksPerPacketField = dos.getClass() + .getDeclaredField("chunksPerPacket"); + chunksPerPacketField.setAccessible(true); + Assert.assertEquals(chunksPerPacketField.getInt(dos), + (finalWritePacketSize - packateMaxHeaderLength) / chunkSize); + + /* get and verify packetSize */ + final Field packetSizeField = dos.getClass() + .getDeclaredField("packetSize"); + packetSizeField.setAccessible(true); + Assert.assertEquals(packetSizeField.getInt(dos), + chunksPerPacketField.getInt(dos) * chunkSize); + } finally { + if (dfsCluster != null) { + dfsCluster.shutdown(); + } + } + } + @Test public void testCongestionBackoff() throws IOException { DfsClientConf dfsClientConf = mock(DfsClientConf.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java index fa3e62cbfcc..1d09a6c1c7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** * This tests read operation of DFS striped file with RS-10-4-64k @@ -29,7 +28,7 @@ public class TestDFSRSDefault10x4StripedInputStream extends TestDFSStripedInputStream { public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.RS_10_4_POLICY_ID); + return SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_10_4_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java index f4dcdf735da..080033d3642 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** * This tests write operation of DFS striped file with RS-10-4-64k @@ -30,7 +29,7 @@ public class TestDFSRSDefault10x4StripedOutputStream @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.RS_10_4_POLICY_ID); + return SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_10_4_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java index 27d3ccfcc3f..5de9dd3434b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** * This tests write operation of DFS striped file with RS-10-4-64k @@ -30,7 +29,7 @@ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.RS_10_4_POLICY_ID); + return SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_10_4_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 68fde95ca8e..e3332fd403e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -97,7 +97,7 @@ public void setup() throws IOException { getEcPolicy().getName()); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } SimulatedFSDataset.setFactory(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index ebdecfd6406..a2d5c8d7b06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -84,7 +84,7 @@ public void setup() throws IOException { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } DFSTestUtil.enableAllECPolicies(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index c66c7f27b71..66a805c43fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -213,7 +213,7 @@ private void setup(Configuration conf) throws IOException { final int numDNs = dataBlocks + parityBlocks; if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } DFSTestUtil.enableAllECPolicies(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 7257bbd793d..14ad6dd8d16 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -513,7 +513,7 @@ public void testHANameNodesWithFederation() throws URISyntaxException { NS2_NN2_HOST); Map> map = - DFSUtil.getHaNnRpcAddresses(conf); + DFSUtilClient.getHaNnRpcAddresses(conf); assertTrue(HAUtil.isHAEnabled(conf, "ns1")); assertTrue(HAUtil.isHAEnabled(conf, "ns2")); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java index de0852810af..d19093da108 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java @@ -16,9 +16,8 @@ * limitations under the License. */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** * This tests read operation of DFS striped file with XOR-2-1-64k erasure code @@ -27,7 +26,7 @@ public class TestDFSXORStripedInputStream extends TestDFSStripedInputStream{ public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.XOR_2_1_POLICY_ID); + return SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java index 658c9ba95d7..ba620b8cc2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** * This tests write operation of DFS striped file with XOR-2-1-64k erasure code @@ -29,7 +28,7 @@ public class TestDFSXORStripedOutputStream extends TestDFSStripedOutputStream{ @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.XOR_2_1_POLICY_ID); + return SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java index c97644ecdc3..d9ab3d57191 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** * This tests write operation of DFS striped file with XOR-2-1-64k erasure code @@ -30,7 +29,7 @@ public class TestDFSXORStripedOutputStreamWithFailure @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.XOR_2_1_POLICY_ID); + return SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 64170780f8c..1f5173234cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -56,6 +56,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FileSystemTestWrapper; +import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -83,6 +84,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.DelegationTokenExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension; @@ -105,8 +107,21 @@ import static org.mockito.Mockito.withSettings; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesEqual; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.apache.hadoop.test.MetricsAsserts.assertGauge; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; @@ -884,7 +899,8 @@ private static void mockCreate(ClientProtocol mcp, .when(mcp) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable) anyObject(), anyBoolean(), - anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject()); + anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(), + anyObject()); } // This test only uses mocks. Called from the end of an existing test to @@ -1668,4 +1684,194 @@ private void verifyShellDeleteWithTrash(FsShell shell, Path path) } } } + + /** This test tests that client will first lookup secrets map + * for key provider uri from {@link Credentials} in + * {@link UserGroupInformation} + * @throws Exception + */ + @Test + public void testProviderUriInCredentials() throws Exception { + String dummyKeyProvider = "dummy://foo:bar@test_provider1"; + DFSClient client = cluster.getFileSystem().getClient(); + Credentials credentials = new Credentials(); + // Key provider uri should be in the secret map of credentials object with + // namenode uri as key + Text lookUpKey = client.getKeyProviderMapKey(); + credentials.addSecretKey(lookUpKey, + DFSUtilClient.string2Bytes(dummyKeyProvider)); + client.ugi.addCredentials(credentials); + client.setKeyProviderUri(null); + Assert.assertEquals("Client Key provider is different from provider in " + + "credentials map", dummyKeyProvider, + client.getKeyProviderUri().toString()); + } + + + /** + * Testing the fallback behavior of keyProviderUri. + * This test tests first the key provider uri is used from conf + * and then used from serverDefaults. + * @throws IOException + */ + @Test + public void testKeyProviderFallBackBehavior() throws IOException { + Configuration clusterConf = cluster.getConfiguration(0); + String dummyKeyProviderUri1 = "dummy://foo:bar@test_provider1"; + // set the key provider uri in conf. + clusterConf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + dummyKeyProviderUri1); + DFSClient mockClient = Mockito.spy(cluster.getFileSystem().getClient()); + mockClient.setKeyProviderUri(null); + // Namenode returning null as keyProviderUri in FSServerDefaults. + FsServerDefaults serverDefaultsWithKeyProviderNull = + getTestServerDefaults(null); + Mockito.doReturn(serverDefaultsWithKeyProviderNull) + .when(mockClient).getServerDefaults(); + Assert.assertEquals( + "Key provider uri from client doesn't match with uri from conf", + dummyKeyProviderUri1, mockClient.getKeyProviderUri().toString()); + Mockito.verify(mockClient, Mockito.times(1)).getServerDefaults(); + + String dummyKeyProviderUri2 = "dummy://foo:bar@test_provider2"; + mockClient.setKeyProviderUri(null); + FsServerDefaults serverDefaultsWithDummyKeyProvider = + getTestServerDefaults(dummyKeyProviderUri2); + // Namenode returning dummyKeyProvider2 in serverDefaults. + Mockito.doReturn(serverDefaultsWithDummyKeyProvider) + .when(mockClient).getServerDefaults(); + Assert.assertEquals( + "Key provider uri from client doesn't match with uri from namenode", + dummyKeyProviderUri2, mockClient.getKeyProviderUri().toString()); + Mockito.verify(mockClient, Mockito.times(2)).getServerDefaults(); + } + + /** + * This test makes sure the client gets the key provider uri from namenode + * instead of its own conf. + * This test assumes both the namenode and client are upgraded. + * @throws Exception + */ + @Test + public void testDifferentKMSProviderOnUpgradedNamenode() throws Exception { + Configuration clusterConf = cluster.getConfiguration(0); + URI namenodeKeyProviderUri = URI.create(getKeyProviderURI()); + Assert.assertEquals("Key Provider for client and namenode are different", + namenodeKeyProviderUri, cluster.getFileSystem().getClient() + .getKeyProviderUri()); + + // Unset the provider path in conf + clusterConf.unset( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); + // Nullify the cached value for key provider uri on client + cluster.getFileSystem().getClient().setKeyProviderUri(null); + // Even after unsetting the local conf, the client key provider should be + // the same as namenode's provider. + Assert.assertEquals("Key Provider for client and namenode are different", + namenodeKeyProviderUri, cluster.getFileSystem().getClient() + .getKeyProviderUri()); + + // Set the provider path to some dummy scheme. + clusterConf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + "dummy://foo:bar@test_provider1"); + // Nullify the cached value for key provider uri on client + cluster.getFileSystem().getClient().setKeyProviderUri(null); + // Even after pointing the conf to some dummy provider, the client key + // provider should be the same as namenode's provider. + Assert.assertEquals("Key Provider for client and namenode are different", + namenodeKeyProviderUri, cluster.getFileSystem().getClient() + .getKeyProviderUri()); + } + + /** + * This test makes sure the client trusts its local conf + * This test assumes the client is upgraded but the namenode is not. + * @throws Exception + */ + @Test + public void testDifferentKMSProviderOnUnUpgradedNamenode() + throws Exception { + Configuration clusterConf = cluster.getConfiguration(0); + URI namenodeKeyProviderUri = URI.create(getKeyProviderURI()); + URI clientKeyProviderUri = + cluster.getFileSystem().getClient().getKeyProviderUri(); + Assert.assertNotNull(clientKeyProviderUri); + // Since the client and the namenode share the same conf, they will have + // identical key provider. + Assert.assertEquals("Key Provider for client and namenode are different", + namenodeKeyProviderUri, clientKeyProviderUri); + + String dummyKeyProviderUri = "dummy://foo:bar@test_provider"; + // Unset the provider path in conf. + clusterConf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + dummyKeyProviderUri); + FsServerDefaults spyServerDefaults = getTestServerDefaults(null); + // Creating a fake serverdefaults so that we can simulate namenode not + // being upgraded. + DFSClient spyClient = Mockito.spy(cluster.getFileSystem().getClient()); + // Clear the cache value of keyProviderUri on client side. + spyClient.setKeyProviderUri(null); + Mockito.doReturn(spyServerDefaults).when(spyClient).getServerDefaults(); + + // Since FsServerDefaults#keyProviderUri is null, the client + // will fallback to local conf which is null. + clientKeyProviderUri = spyClient.getKeyProviderUri(); + Assert.assertEquals("Client keyProvider should be " + dummyKeyProviderUri, + dummyKeyProviderUri, clientKeyProviderUri.toString()); + Mockito.verify(spyClient, Mockito.times(1)).getServerDefaults(); + } + + // Given a provider uri return serverdefaults. + // provider uri == null means the namenode does not support returning + // provider uri in FSServerDefaults object. + private FsServerDefaults getTestServerDefaults(String providerPath) { + FsServerDefaults serverDefaults = new FsServerDefaults( + conf.getLongBytes(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT), + conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, DFS_BYTES_PER_CHECKSUM_DEFAULT), + conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, + DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT), + (short) conf.getInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT), + conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), + conf.getBoolean( + DFS_ENCRYPT_DATA_TRANSFER_KEY, DFS_ENCRYPT_DATA_TRANSFER_DEFAULT), + conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT), + DataChecksum.Type.valueOf(DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT), + providerPath); + return serverDefaults; + } + + /** + * This test performs encrypted read/write and picks up the key provider uri + * from the credentials and not the conf. + * @throws Exception + */ + @Test + public void testEncryptedReadWriteUsingDiffKeyProvider() throws Exception { + final HdfsAdmin dfsAdmin = + new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); + Configuration clusterConf = cluster.getConfiguration(0); + clusterConf.unset( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); + DFSClient client = cluster.getFileSystem().getClient(); + Credentials credentials = new Credentials(); + Text lookUpKey = client.getKeyProviderMapKey(); + credentials.addSecretKey(lookUpKey, + DFSUtilClient.string2Bytes(getKeyProviderURI())); + client.ugi.addCredentials(credentials); + // Create a base file for comparison + final Path baseFile = new Path("/base"); + final int len = 8192; + DFSTestUtil.createFile(fs, baseFile, len, (short) 1, 0xFEED); + // Create the first enc file + final Path zone = new Path("/zone"); + fs.mkdirs(zone); + dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH); + final Path encFile1 = new Path(zone, "myfile"); + DFSTestUtil.createFile(fs, encFile1, len, (short) 1, 0xFEED); + // Read them back in and compare byte-by-byte + verifyFilesEqual(fs, baseFile, encFile1, len); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 5ba4403d2ed..1aee929d83a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -23,9 +23,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.client.HdfsAdmin; @@ -34,6 +34,7 @@ import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -44,8 +45,8 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.security.PrivilegedExceptionAction; -import java.util.Arrays; import java.util.Collection; +import java.util.List; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.junit.Assert.*; @@ -302,12 +303,12 @@ public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Excepti @Test public void testGetErasureCodingPolicy() throws Exception { - ErasureCodingPolicy[] sysECPolicies = - ErasureCodingPolicyManager.getSystemPolicies(); + List sysECPolicies = + SystemErasureCodingPolicies.getPolicies(); assertTrue("System ecPolicies should exist", - sysECPolicies.length > 0); + sysECPolicies.size() > 0); - ErasureCodingPolicy usingECPolicy = sysECPolicies[0]; + ErasureCodingPolicy usingECPolicy = sysECPolicies.get(0); String src = "/ec2"; final Path ecDir = new Path(src); fs.mkdir(ecDir, FsPermission.getDirDefault()); @@ -353,12 +354,10 @@ public void testSetInvalidPolicy() @Test public void testGetAllErasureCodingPolicies() throws Exception { - ErasureCodingPolicy[] sysECPolicies = ErasureCodingPolicyManager - .getSystemPolicies(); Collection allECPolicies = fs .getAllErasureCodingPolicies(); assertTrue("All system policies should be enabled", - allECPolicies.containsAll(Arrays.asList(sysECPolicies))); + allECPolicies.containsAll(SystemErasureCodingPolicies.getPolicies())); } @Test @@ -383,9 +382,9 @@ public void testGetErasureCodingPolicyOnANonExistentFile() throws Exception { @Test public void testMultiplePoliciesCoExist() throws Exception { - ErasureCodingPolicy[] sysPolicies = - ErasureCodingPolicyManager.getSystemPolicies(); - if (sysPolicies.length > 1) { + List sysPolicies = + SystemErasureCodingPolicies.getPolicies(); + if (sysPolicies.size() > 1) { for (ErasureCodingPolicy policy : sysPolicies) { Path dir = new Path("/policy_" + policy.getId()); fs.mkdir(dir, FsPermission.getDefault()); @@ -425,8 +424,8 @@ public HdfsAdmin run() throws Exception { Path ecfile = new Path(ecdir, "ecfile"); fs.setPermission(new Path("/"), new FsPermission((short)0777)); userfs.mkdirs(ecdir); - final String ecPolicyName = - ErasureCodingPolicyManager.getSystemPolicies()[0].getName(); + final String ecPolicyName = StripedFileTestUtil.getDefaultECPolicy() + .getName(); useradmin.setErasureCodingPolicy(ecdir, ecPolicyName); assertEquals("Policy not present on dir", ecPolicyName, @@ -521,4 +520,48 @@ public HdfsAdmin run() throws Exception { noadmin.getErasureCodingPolicies(); superadmin.getErasureCodingPolicies(); } + + /** + * Test apply specific erasure coding policy on single file. Usually file's + * policy is inherited from its parent. + */ + @Test + public void testFileLevelECPolicy() throws Exception { + final Path dirPath = new Path("/striped"); + final Path filePath0 = new Path(dirPath, "file0"); + final Path filePath1 = new Path(dirPath, "file1"); + + fs.mkdirs(dirPath); + fs.setErasureCodingPolicy(dirPath, EC_POLICY.getName()); + + // null EC policy name value means inheriting parent directory's policy + fs.newFSDataOutputStreamBuilder(filePath0).build().close(); + ErasureCodingPolicy ecPolicyOnFile = fs.getErasureCodingPolicy(filePath0); + assertEquals(EC_POLICY, ecPolicyOnFile); + + // Test illegal EC policy name + final String illegalPolicyName = "RS-DEFAULT-1-2-64k"; + try { + fs.newFSDataOutputStreamBuilder(filePath1) + .setEcPolicyName(illegalPolicyName).build().close(); + Assert.fail("illegal erasure coding policy should not be found"); + } catch (Exception e) { + GenericTestUtils.assertExceptionContains("Policy '" + illegalPolicyName + + "' does not match any enabled erasure coding policies", e); + } + fs.delete(dirPath, true); + + // Test create a file with a different EC policy than its parent directory + fs.mkdirs(dirPath); + final ErasureCodingPolicy ecPolicyOnDir = + SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_3_2_POLICY_ID); + ecPolicyOnFile = EC_POLICY; + fs.setErasureCodingPolicy(dirPath, ecPolicyOnDir.getName()); + fs.newFSDataOutputStreamBuilder(filePath0) + .setEcPolicyName(ecPolicyOnFile.getName()).build().close(); + assertEquals(ecPolicyOnFile, fs.getErasureCodingPolicy(filePath0)); + assertEquals(ecPolicyOnDir, fs.getErasureCodingPolicy(dirPath)); + fs.delete(dirPath, true); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index e47c8b13e1c..9dff529ef71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -16,6 +16,7 @@ * limitations under the License. */ package org.apache.hadoop.hdfs; + import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; @@ -171,6 +172,7 @@ public void testServerDefaults() throws IOException { assertEquals(DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT, serverDefaults.getWritePacketSize()); assertEquals(DFS_REPLICATION_DEFAULT + 1, serverDefaults.getReplication()); assertEquals(IO_FILE_BUFFER_SIZE_DEFAULT, serverDefaults.getFileBufferSize()); + assertEquals(7, serverDefaults.getDefaultStoragePolicyId()); } finally { fs.close(); cluster.shutdown(); @@ -1141,7 +1143,7 @@ private void doCreateTest(CreationMethod method) throws Exception { try { nnrpc.create(pathStr, new FsPermission((short)0755), "client", new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), - true, (short)1, 128*1024*1024L, null); + true, (short)1, 128*1024*1024L, null, null); fail("Should have thrown exception when creating '" + pathStr + "'" + " by " + method); } catch (InvalidPathException ipe) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java index 0868b5441e5..9fc6b389494 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java @@ -96,29 +96,42 @@ public void testCache() throws Exception { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://foo:bar@test_provider1"); - KeyProvider keyProvider1 = kpCache.get(conf); + KeyProvider keyProvider1 = kpCache.get(conf, + getKeyProviderUriFromConf(conf)); Assert.assertNotNull("Returned Key Provider is null !!", keyProvider1); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://foo:bar@test_provider1"); - KeyProvider keyProvider2 = kpCache.get(conf); + KeyProvider keyProvider2 = kpCache.get(conf, + getKeyProviderUriFromConf(conf)); Assert.assertTrue("Different KeyProviders returned !!", keyProvider1 == keyProvider2); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://test_provider3"); - KeyProvider keyProvider3 = kpCache.get(conf); + KeyProvider keyProvider3 = kpCache.get(conf, + getKeyProviderUriFromConf(conf)); Assert.assertFalse("Same KeyProviders returned !!", keyProvider1 == keyProvider3); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://hello:there@test_provider1"); - KeyProvider keyProvider4 = kpCache.get(conf); + KeyProvider keyProvider4 = kpCache.get(conf, + getKeyProviderUriFromConf(conf)); Assert.assertFalse("Same KeyProviders returned !!", keyProvider1 == keyProvider4); } + + private URI getKeyProviderUriFromConf(Configuration conf) { + String providerUriStr = conf.get( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); + if (providerUriStr == null || providerUriStr.isEmpty()) { + return null; + } + return URI.create(providerUriStr); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java index 20596c56b95..16cdf9b322d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java @@ -363,7 +363,8 @@ public void testFactory() throws Exception { .when(mcp) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable) anyObject(), anyBoolean(), - anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject()); + anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(), + anyObject()); final Configuration conf = new Configuration(); final DFSClient c1 = createDFSClientAs(ugi[0], conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index affb541891f..2bd4a905bce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -99,7 +99,7 @@ public void setup() throws IOException { false); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index 32f6bc8013c..6379db5ed1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -21,9 +21,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; @@ -68,7 +67,7 @@ public void setup() throws IOException { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } DFSTestUtil.enableAllECPolicies(conf); @@ -138,8 +137,8 @@ public void testNestedEcPolicy() throws Exception { final Path ec63FilePath = new Path(childDir, "ec_6_3_file"); final Path ec32FilePath = new Path(childDir, "ec_3_2_file"); final Path ec63FilePath2 = new Path(childDir, "ec_6_3_file_2"); - final ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager - .getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID); + final ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); fs.mkdirs(parentDir); fs.setErasureCodingPolicy(parentDir, ecPolicy.getName()); @@ -236,8 +235,8 @@ public void testChangeRootDirEcPolicy() throws Exception { final Path rootPath = new Path("/"); final Path ec63FilePath = new Path(rootPath, "ec_6_3_file"); final Path ec32FilePath = new Path(rootPath, "ec_3_2_file"); - final ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager - .getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID); + final ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); fs.unsetErasureCodingPolicy(rootPath); fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 314ecf62c4b..7647ac45fb1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -17,9 +17,14 @@ */ package org.apache.hadoop.hdfs.protocolPB; + +import com.google.protobuf.UninitializedMessageException; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; + import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -35,9 +40,12 @@ import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.StripedFileTestUtil; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -97,8 +105,10 @@ import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.io.Text; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DataChecksum; import org.junit.Assert; import org.junit.Test; @@ -868,5 +878,110 @@ private void assertDnInfosEqual(DatanodeInfo[] dnInfos1, } } + /** + * Test case for old namenode where the namenode doesn't support returning + * keyProviderUri. + */ + @Test + public void testFSServerDefaultsHelper() { + HdfsProtos.FsServerDefaultsProto.Builder b = + HdfsProtos.FsServerDefaultsProto + .newBuilder(); + b.setBlockSize(DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); + b.setBytesPerChecksum(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT); + b.setWritePacketSize( + HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT); + b.setReplication(DFSConfigKeys.DFS_REPLICATION_DEFAULT); + b.setFileBufferSize(DFSConfigKeys.IO_FILE_BUFFER_SIZE_DEFAULT); + b.setEncryptDataTransfer(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEFAULT); + b.setTrashInterval(DFSConfigKeys.FS_TRASH_INTERVAL_DEFAULT); + b.setChecksumType(HdfsProtos.ChecksumTypeProto.valueOf( + DataChecksum.Type.valueOf(DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT).id)); + HdfsProtos.FsServerDefaultsProto proto = b.build(); + assertFalse("KeyProvider uri is not supported", + proto.hasKeyProviderUri()); + FsServerDefaults fsServerDefaults = PBHelperClient.convert(proto); + Assert.assertNotNull("FsServerDefaults is null", fsServerDefaults); + Assert.assertNull("KeyProviderUri should be null", + fsServerDefaults.getKeyProviderUri()); + } + + @Test + public void testConvertErasureCodingPolicy() throws Exception { + // Check conversion of the built-in policies. + for (ErasureCodingPolicy policy : + SystemErasureCodingPolicies.getPolicies()) { + HdfsProtos.ErasureCodingPolicyProto proto = PBHelperClient + .convertErasureCodingPolicy(policy); + // Optional fields should not be set. + assertFalse("Unnecessary field is set.", proto.hasName()); + assertFalse("Unnecessary field is set.", proto.hasSchema()); + assertFalse("Unnecessary field is set.", proto.hasCellSize()); + // Convert proto back to an object and check for equality. + ErasureCodingPolicy convertedPolicy = PBHelperClient + .convertErasureCodingPolicy(proto); + assertEquals("Converted policy not equal", policy, convertedPolicy); + } + // Check conversion of a non-built-in policy. + ECSchema newSchema = new ECSchema("testcodec", 3, 2); + ErasureCodingPolicy newPolicy = + new ErasureCodingPolicy(newSchema, 128 * 1024, (byte) 254); + HdfsProtos.ErasureCodingPolicyProto proto = PBHelperClient + .convertErasureCodingPolicy(newPolicy); + // Optional fields should be set. + assertTrue("Optional field not set", proto.hasName()); + assertTrue("Optional field not set", proto.hasSchema()); + assertTrue("Optional field not set", proto.hasCellSize()); + ErasureCodingPolicy convertedPolicy = PBHelperClient + .convertErasureCodingPolicy(proto); + // Converted policy should be equal. + assertEquals("Converted policy not equal", newPolicy, convertedPolicy); + } + + @Test(expected = UninitializedMessageException.class) + public void testErasureCodingPolicyMissingId() throws Exception { + HdfsProtos.ErasureCodingPolicyProto.Builder builder = + HdfsProtos.ErasureCodingPolicyProto.newBuilder(); + PBHelperClient.convertErasureCodingPolicy(builder.build()); + } + + @Test + public void testErasureCodingPolicyMissingOptionalFields() throws Exception { + // For non-built-in policies, the optional fields are required + // when parsing an ErasureCodingPolicyProto. + HdfsProtos.ECSchemaProto schemaProto = + PBHelperClient.convertECSchema( + StripedFileTestUtil.getDefaultECPolicy().getSchema()); + try { + PBHelperClient.convertErasureCodingPolicy( + HdfsProtos.ErasureCodingPolicyProto.newBuilder() + .setId(14) + .setSchema(schemaProto) + .setCellSize(123) + .build()); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains("Missing", e); + } + try { + PBHelperClient.convertErasureCodingPolicy( + HdfsProtos.ErasureCodingPolicyProto.newBuilder() + .setId(14) + .setName("testpolicy") + .setCellSize(123) + .build()); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains("Missing", e); + } + try { + PBHelperClient.convertErasureCodingPolicy( + HdfsProtos.ErasureCodingPolicyProto.newBuilder() + .setId(14) + .setName("testpolicy") + .setSchema(schemaProto) + .build()); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains("Missing", e); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index b9220e0f6c3..aa47eeb3e3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -412,7 +412,7 @@ private BPOfferService setupBPOSForNNs(DataNode mockDn, Mockito.eq(new InetSocketAddress(port))); } - return new BPOfferService(Lists.newArrayList(nnMap.keySet()), + return new BPOfferService("test_ns", Lists.newArrayList(nnMap.keySet()), Collections.nCopies(nnMap.size(), null), mockDn); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java index 8bf40e30fa4..560b32e0ee3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java @@ -51,7 +51,9 @@ public void setupBPM() { bpm = new BlockPoolManager(mockDN){ @Override - protected BPOfferService createBPOS(List nnAddrs, + protected BPOfferService createBPOS( + final String nameserviceId, + List nnAddrs, List lifelineNnAddrs) { final int idx = mockIdx++; doLog("create #" + idx); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index e34837c7b72..579252bd381 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -80,7 +80,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.BlockRecord; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringStripedBlock; @@ -803,8 +802,7 @@ public void testRURReplicas() throws Exception { @Test(timeout=60000) public void testSafeLength() throws Exception { // hard coded policy to work with hard coded test suite - ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager - .getSystemPolicies()[0]; + ErasureCodingPolicy ecPolicy = StripedFileTestUtil.getDefaultECPolicy(); RecoveringStripedBlock rBlockStriped = new RecoveringStripedBlock(rBlock, new byte[9], ecPolicy); BlockRecoveryWorker recoveryWorker = new BlockRecoveryWorker(dn); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java index dc091ed31b9..953de4f7a33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java @@ -94,7 +94,7 @@ public VolumeCheckResult answer( return volume; } - @Test (timeout = 1000) + @Test (timeout = 300000) public void testDiskCheckTimeout() throws Exception { LOG.info("Executing {}", testName.getMethodName()); final FsVolumeSpi volume = makeSlowVolume(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java index 00b1af2a11a..4ed6371afbd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java @@ -20,8 +20,6 @@ import com.google.common.base.Optional; import com.google.common.base.Supplier; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.FakeTimer; @@ -29,12 +27,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.junit.Assert.assertFalse; @@ -93,35 +88,9 @@ public void testScheduler() throws Exception { waitTestCheckableCheckCount(target2, 2L); } - @Test (timeout=60000) - public void testCancellation() throws Exception { - LatchedCheckable target = new LatchedCheckable(); - final FakeTimer timer = new FakeTimer(); - final LatchedCallback callback = new LatchedCallback(target); - ThrottledAsyncChecker checker = - new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, - getExecutorService()); - - Optional> olf = - checker.schedule(target, true); - if (olf.isPresent()) { - Futures.addCallback(olf.get(), callback); - } - - // Request immediate cancellation. - checker.shutdownAndWait(0, TimeUnit.MILLISECONDS); - try { - assertFalse(olf.get().get()); - fail("Failed to get expected InterruptedException"); - } catch (ExecutionException ee) { - assertTrue(ee.getCause() instanceof InterruptedException); - } - callback.failureLatch.await(); - } - @Test (timeout=60000) public void testConcurrentChecks() throws Exception { - LatchedCheckable target = new LatchedCheckable(); + final StalledCheckable target = new StalledCheckable(); final FakeTimer timer = new FakeTimer(); ThrottledAsyncChecker checker = new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, @@ -136,25 +105,6 @@ public void testConcurrentChecks() throws Exception { // for the first caller. assertTrue(olf1.isPresent()); assertFalse(olf2.isPresent()); - - // Unblock the latch and wait for it to finish execution. - target.latch.countDown(); - olf1.get().get(); - - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - // We should get an absent Optional. - // This can take a short while until the internal callback in - // ThrottledAsyncChecker is scheduled for execution. - // Also this should not trigger a new check operation as the timer - // was not advanced. If it does trigger a new check then the test - // will fail with a timeout. - final Optional> olf3 = - checker.schedule(target, true); - return !olf3.isPresent(); - } - }, 100, 10000); } /** @@ -191,6 +141,32 @@ public Boolean get() { } }, 100, 10000); } + + /** + * Ensure that an exception thrown by the check routine is + * propagated. + * + * @throws Exception + */ + @Test(timeout=60000) + public void testExceptionIsPropagated() throws Exception { + final ThrowingCheckable target = new ThrowingCheckable(); + final FakeTimer timer = new FakeTimer(); + ThrottledAsyncChecker checker = + new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, + getExecutorService()); + + final Optional> olf = + checker.schedule(target, true); + assertTrue(olf.isPresent()); + try { + olf.get().get(); + fail("Failed to get expected ExecutionException"); + } catch(ExecutionException ee) { + assertTrue(ee.getCause() instanceof DummyException); + } + } + /** * Ensure that the exception from a failed check is cached * and returned without re-running the check when the minimum @@ -245,6 +221,9 @@ public Boolean check(Boolean context) { } } + /** + * A Checkable that throws an exception when checked. + */ private static class ThrowingCheckable extends TestCheckableBase { @Override @@ -258,43 +237,14 @@ private static class DummyException extends Exception { } /** - * A checkable that hangs until signaled. + * A checkable that hangs forever when checked. */ - private static class LatchedCheckable + private static class StalledCheckable implements Checkable { - private final CountDownLatch latch = new CountDownLatch(1); - @Override public Boolean check(Boolean ignored) throws InterruptedException { - LOG.info("LatchedCheckable {} waiting.", this); - latch.await(); - return true; // Unreachable. - } - } - - /** - * A {@link FutureCallback} that counts its invocations. - */ - private static final class LatchedCallback - implements FutureCallback { - private final CountDownLatch successLatch = new CountDownLatch(1); - private final CountDownLatch failureLatch = new CountDownLatch(1); - private final Checkable target; - - private LatchedCallback(Checkable target) { - this.target = target; - } - - @Override - public void onSuccess(@Nonnull Boolean result) { - LOG.info("onSuccess callback invoked for {}", target); - successLatch.countDown(); - } - - @Override - public void onFailure(@Nonnull Throwable t) { - LOG.info("onFailure callback invoked for {} with exception", target, t); - failureLatch.countDown(); + Thread.sleep(Long.MAX_VALUE); + return false; // Unreachable. } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java index 52cab57dc6f..b0993b5b32d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java @@ -17,6 +17,14 @@ */ package org.apache.hadoop.hdfs.server.datanode.checker; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.anySet; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; + import com.google.common.base.Optional; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -35,12 +43,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Matchers.anySet; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import org.junit.rules.Timeout; import org.slf4j.LoggerFactory; import java.util.Set; @@ -58,10 +61,10 @@ public class TestThrottledAsyncCheckerTimeout { @Rule public TestName testName = new TestName(); + @Rule + public Timeout testTimeout = new Timeout(300_000); - Configuration conf; private static final long DISK_CHECK_TIMEOUT = 10; - private static final long DISK_CHECK_TIME = 100; private ReentrantLock lock; private ExecutorService getExecutorService() { @@ -73,7 +76,7 @@ public void initializeLock() { lock = new ReentrantLock(); } - @Test (timeout = 1000) + @Test public void testDiskCheckTimeout() throws Exception { LOG.info("Executing {}", testName.getMethodName()); @@ -123,7 +126,7 @@ public void onFailure(Throwable t) { assertTrue(throwable[0] instanceof TimeoutException); } - @Test (timeout = 2000) + @Test public void testDiskCheckTimeoutInvokesOneCallbackOnly() throws Exception { LOG.info("Executing {}", testName.getMethodName()); @@ -143,13 +146,12 @@ public void testDiskCheckTimeoutInvokesOneCallbackOnly() throws Exception { assertTrue(olf1.isPresent()); Futures.addCallback(olf1.get(), futureCallback); - // Wait for the callback - Thread.sleep(DISK_CHECK_TIMEOUT); - // Verify that timeout results in only 1 onFailure call and 0 onSuccess // calls. - verify(futureCallback, times(1)).onFailure(any()); - verify(futureCallback, times(0)).onSuccess(any()); + verify(futureCallback, timeout((int) DISK_CHECK_TIMEOUT*10).times(1)) + .onFailure(any()); + verify(futureCallback, timeout((int) DISK_CHECK_TIMEOUT*10).times(0)) + .onSuccess(any()); // Release lock so that target can acquire it. lock.unlock(); @@ -160,16 +162,15 @@ public void testDiskCheckTimeoutInvokesOneCallbackOnly() throws Exception { assertTrue(olf2.isPresent()); Futures.addCallback(olf2.get(), futureCallback); - // Wait for the callback - Thread.sleep(DISK_CHECK_TIME); - // Verify that normal check (dummy) results in only 1 onSuccess call. - // Number of times onFailure is invoked should remain the same - 1. - verify(futureCallback, times(1)).onFailure(any()); - verify(futureCallback, times(1)).onSuccess(any()); + // Number of times onFailure is invoked should remain the same i.e. 1. + verify(futureCallback, timeout((int) DISK_CHECK_TIMEOUT*10).times(1)) + .onFailure(any()); + verify(futureCallback, timeout((int) DISK_CHECK_TIMEOUT*10).times(1)) + .onSuccess(any()); } - @Test (timeout = 1000) + @Test public void testTimeoutExceptionIsNotThrownForGoodDisk() throws Exception { LOG.info("Executing {}", testName.getMethodName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index 9af61aedba9..707d46fd3cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -822,6 +822,48 @@ public void testMoverFailedRetryWithPinnedBlocks() throws Exception { } } + @Test(timeout = 300000) + public void testMoverWhenStoragePolicyUnset() throws Exception { + final Configuration conf = new HdfsConfiguration(); + initConf(conf); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1) + .storageTypes( + new StorageType[][] {{StorageType.DISK, StorageType.ARCHIVE}}) + .build(); + try { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final String file = "/testMoverWhenStoragePolicyUnset"; + // write to DISK + DFSTestUtil.createFile(dfs, new Path(file), 1L, (short) 1, 0L); + + // move to ARCHIVE + dfs.setStoragePolicy(new Path(file), "COLD"); + int rc = ToolRunner.run(conf, new Mover.Cli(), + new String[] {"-p", file.toString()}); + Assert.assertEquals("Movement to ARCHIVE should be successful", 0, rc); + + // Wait till namenode notified about the block location details + waitForLocatedBlockWithArchiveStorageType(dfs, file, 1); + + // verify before unset policy + LocatedBlock lb = dfs.getClient().getLocatedBlocks(file, 0).get(0); + Assert.assertTrue(StorageType.ARCHIVE == (lb.getStorageTypes())[0]); + + // unset storage policy + dfs.unsetStoragePolicy(new Path(file)); + rc = ToolRunner.run(conf, new Mover.Cli(), + new String[] {"-p", file.toString()}); + Assert.assertEquals("Movement to DISK should be successful", 0, rc); + + lb = dfs.getClient().getLocatedBlocks(file, 0).get(0); + Assert.assertTrue(StorageType.DISK == (lb.getStorageTypes())[0]); + } finally { + cluster.shutdown(); + } + } + private void createFileWithFavoredDatanodes(final Configuration conf, final MiniDFSCluster cluster, final DistributedFileSystem dfs) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index c1f0a7be3f0..3a3c47177aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -587,14 +587,16 @@ long executeOp(int daemonId, int inputIdx, String clientName) throws IOException { long start = Time.now(); // dummyActionNoSynch(fileIdx); - clientProto.create(fileNames[daemonId][inputIdx], FsPermission.getDefault(), - clientName, new EnumSetWritable(EnumSet - .of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, - replication, BLOCK_SIZE, CryptoProtocolVersion.supported()); + clientProto.create(fileNames[daemonId][inputIdx], + FsPermission.getDefault(), clientName, + new EnumSetWritable(EnumSet + .of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, + replication, BLOCK_SIZE, CryptoProtocolVersion.supported(), null); long end = Time.now(); - for(boolean written = !closeUponCreate; !written; + for (boolean written = !closeUponCreate; !written; written = clientProto.complete(fileNames[daemonId][inputIdx], - clientName, null, HdfsConstants.GRANDFATHER_INODE_ID)); + clientName, null, HdfsConstants.GRANDFATHER_INODE_ID)) { + }; return end-start; } @@ -1139,7 +1141,7 @@ void generateInputs(int[] ignore) throws IOException { String fileName = nameGenerator.getNextFileName("ThroughputBench"); clientProto.create(fileName, FsPermission.getDefault(), clientName, new EnumSetWritable(EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, replication, - BLOCK_SIZE, CryptoProtocolVersion.supported()); + BLOCK_SIZE, CryptoProtocolVersion.supported(), null); ExtendedBlock lastBlock = addBlocks(fileName, clientName); clientProto.complete(fileName, clientName, lastBlock, HdfsConstants.GRANDFATHER_INODE_ID); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java index 94abe3ef837..1aa77266f0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java @@ -85,7 +85,7 @@ public void testRetryAddBlockWhileInChooseTarget() throws Exception { nn.create(src, FsPermission.getFileDefault(), "clientName", new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), - true, (short)3, 1024, null); + true, (short)3, 1024, null, null); // start first addBlock() LOG.info("Starting first addBlock for " + src); @@ -157,7 +157,7 @@ public void testAddBlockRetryShouldReturnBlockWithLocations() // create file nameNodeRpc.create(src, FsPermission.getFileDefault(), "clientName", new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, - (short) 3, 1024, null); + (short) 3, 1024, null, null); // start first addBlock() LOG.info("Starting first addBlock for " + src); LocatedBlock lb1 = nameNodeRpc.addBlock(src, "clientName", null, null, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java index f40c464e24d..7cef64be225 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java @@ -109,7 +109,7 @@ private void doTestChooseTargetNormalCase() throws Exception { // Create the file with client machine HdfsFileStatus fileStatus = namesystem.startFile(src, perm, clientMachine, clientMachine, EnumSet.of(CreateFlag.CREATE), true, - replication, DEFAULT_BLOCK_SIZE, null, false); + replication, DEFAULT_BLOCK_SIZE, null, null, false); //test chooseTarget for new file LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine, @@ -139,7 +139,7 @@ private void doTestChooseTargetSpecialCase() throws Exception { // Create the file with client machine HdfsFileStatus fileStatus = namesystem.startFile(src, perm, clientMachine, clientMachine, EnumSet.of(CreateFlag.CREATE), true, - (short) 20, DEFAULT_BLOCK_SIZE, null, false); + (short) 20, DEFAULT_BLOCK_SIZE, null, null, false); //test chooseTarget for new file LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java index 1a10b7a5351..0931ff44ade 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDefaultBlockPlacementPolicy.java @@ -138,7 +138,7 @@ private void testPlacement(String clientMachine, // Create the file with client machine HdfsFileStatus fileStatus = namesystem.startFile(src, perm, clientMachine, clientMachine, EnumSet.of(CreateFlag.CREATE), true, - REPLICATION_FACTOR, DEFAULT_BLOCK_SIZE, null, false); + REPLICATION_FACTOR, DEFAULT_BLOCK_SIZE, null, null, false); LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine, null, null, fileStatus.getFileId(), null, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java index 4b4b196a06c..e35fa119ecb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.StripedFileTestUtil; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -41,9 +42,6 @@ */ public class TestEnabledECPolicies { - private static final ErasureCodingPolicy[] SYSTEM_POLICIES = - ErasureCodingPolicyManager.getSystemPolicies(); - @Rule public Timeout testTimeout = new Timeout(60000); @@ -112,13 +110,16 @@ public void testGetPolicies() throws Exception { testGetPolicies(enabledPolicies); // Enable one policy - enabledPolicies = new ErasureCodingPolicy[] - {SYSTEM_POLICIES[1]}; + enabledPolicies = new ErasureCodingPolicy[]{ + SystemErasureCodingPolicies.getPolicies().get(1) + }; testGetPolicies(enabledPolicies); // Enable two policies - enabledPolicies = new ErasureCodingPolicy[] - {SYSTEM_POLICIES[1], SYSTEM_POLICIES[2]}; + enabledPolicies = new ErasureCodingPolicy[]{ + SystemErasureCodingPolicies.getPolicies().get(1), + SystemErasureCodingPolicies.getPolicies().get(2) + }; testGetPolicies(enabledPolicies); } @@ -145,7 +146,7 @@ private void testGetPolicies(ErasureCodingPolicy[] enabledPolicies) } Assert.assertEquals(enabledPolicies.length, found.size()); // Check that getEnabledPolicyByName only returns enabled policies - for (ErasureCodingPolicy p: SYSTEM_POLICIES) { + for (ErasureCodingPolicy p: SystemErasureCodingPolicies.getPolicies()) { if (found.contains(p.getName())) { // Enabled policy should be present Assert.assertNotNull( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index e1861643f3f..22c40fb2e1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; @@ -77,8 +78,8 @@ public class TestFSImage { private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ = "image-with-zero-block-size.tar.gz"; private static final ErasureCodingPolicy testECPolicy = - ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.RS_10_4_POLICY_ID); + SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_10_4_POLICY_ID); @Test public void testPersist() throws IOException { @@ -470,8 +471,8 @@ public void testSupportBlockGroup() throws Exception { DistributedFileSystem fs = cluster.getFileSystem(); Path parentDir = new Path("/ec-10-4"); Path childDir = new Path(parentDir, "ec-3-2"); - ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager - .getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID); + ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); // Create directories and files fs.mkdirs(parentDir); @@ -519,8 +520,8 @@ public void testSupportBlockGroup() throws Exception { // check the information of file_3_2 inode = fsn.dir.getINode(file_3_2.toString()).asFile(); assertTrue(inode.isStriped()); - assertEquals(ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.RS_3_2_POLICY_ID).getId(), + assertEquals(SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_3_2_POLICY_ID).getId(), inode.getErasureCodingPolicyID()); blks = inode.getBlocks(); assertEquals(1, blks.length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index ced6c7fc67c..22463573432 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -107,7 +108,9 @@ INodeFile createINodeFile(short replication, long preferredBlockSize) { INodeFile createStripedINodeFile(long preferredBlockSize) { return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, - null, null, HdfsConstants.RS_6_3_POLICY_ID, preferredBlockSize, + null, null, + StripedFileTestUtil.getDefaultECPolicy().getId(), + preferredBlockSize, HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED); } @@ -140,7 +143,7 @@ public void testContiguousLayoutRedundancy() { try { new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, new Short((short) 3) /*replication*/, - HdfsConstants.RS_6_3_POLICY_ID /*ec policy*/, + StripedFileTestUtil.getDefaultECPolicy().getId() /*ec policy*/, preferredBlockSize, HdfsConstants.WARM_STORAGE_POLICY_ID, CONTIGUOUS); fail("INodeFile construction should fail when both replication and " + "ECPolicy requested!"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java index d7a2c811a59..d217813bd5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java @@ -223,15 +223,20 @@ public void testCreate() throws Exception { // Two retried calls succeed newCall(); HdfsFileStatus status = nnRpc.create(src, perm, "holder", - new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, - (short) 1, BlockSize, null); - Assert.assertEquals(status, nnRpc.create(src, perm, "holder", new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, (short) 1, BlockSize, null)); - Assert.assertEquals(status, nnRpc.create(src, perm, "holder", new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, (short) 1, BlockSize, null)); - + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, + (short) 1, BlockSize, null, null); + Assert.assertEquals(status, nnRpc.create(src, perm, "holder", + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, + (short) 1, BlockSize, null, null)); + Assert.assertEquals(status, nnRpc.create(src, perm, "holder", + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, + (short) 1, BlockSize, null, null)); // A non-retried call fails newCall(); try { - nnRpc.create(src, perm, "holder", new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, (short) 1, BlockSize, null); + nnRpc.create(src, perm, "holder", + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), + true, (short) 1, BlockSize, null, null); Assert.fail("testCreate - expected exception is not thrown"); } catch (IOException e) { // expected diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java index 5da19a713ca..8c2acf6873d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java @@ -41,6 +41,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; @@ -48,10 +49,13 @@ import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.LogVerificationAppender; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; @@ -564,7 +568,58 @@ public void testCorruptImageFallback() throws IOException { } finally { cluster.shutdown(); } -} + } + + @Test(timeout=30000) + public void testCorruptImageFallbackLostECPolicy() throws IOException { + final ErasureCodingPolicy defaultPolicy = StripedFileTestUtil + .getDefaultECPolicy(); + final String policy = defaultPolicy.getName(); + final Path f1 = new Path("/f1"); + config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, policy); + + MiniDFSCluster cluster = new MiniDFSCluster.Builder(config) + .numDataNodes(0) + .format(true) + .build(); + try { + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + // set root directory to use the default ec policy + Path srcECDir = new Path("/"); + fs.setErasureCodingPolicy(srcECDir, + defaultPolicy.getName()); + + // create a file which will use the default ec policy + fs.create(f1); + FileStatus fs1 = fs.getFileStatus(f1); + assertTrue(fs1.isErasureCoded()); + ErasureCodingPolicy fs1Policy = fs.getErasureCodingPolicy(f1); + assertEquals(fs1Policy, defaultPolicy); + } finally { + cluster.close(); + } + + // Delete a single md5sum + corruptFSImageMD5(false); + // Should still be able to start + cluster = new MiniDFSCluster.Builder(config) + .numDataNodes(0) + .format(false) + .build(); + try { + cluster.waitActive(); + ErasureCodingPolicy[] ecPolicies = cluster.getNameNode() + .getNamesystem().getErasureCodingPolicyManager().getEnabledPolicies(); + DistributedFileSystem fs = cluster.getFileSystem(); + // make sure the ec policy of the file is still correct + assertEquals(fs.getErasureCodingPolicy(f1), defaultPolicy); + // make sure after fsimage fallback, enabled ec policies are not cleared. + assertTrue(ecPolicies.length == 1); + } finally { + cluster.shutdown(); + } + } /** * This test tests hosts include list contains host names. After namenode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index 2c6390011a7..d2f41bec288 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -74,14 +74,14 @@ public class TestStripedINodeFile { // use hard coded policy - see HDFS-9816 private static final ErasureCodingPolicy testECPolicy - = ErasureCodingPolicyManager.getSystemPolicies()[0]; + = StripedFileTestUtil.getDefaultECPolicy(); @Rule public Timeout globalTimeout = new Timeout(300000); private static INodeFile createStripedINodeFile() { return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, - null, null, HdfsConstants.RS_6_3_POLICY_ID, 1024L, + null, null, StripedFileTestUtil.getDefaultECPolicy().getId(), 1024L, HdfsConstants.COLD_STORAGE_POLICY_ID, BlockType.STRIPED); } @@ -118,7 +118,7 @@ public void testStripedLayoutRedundancy() { try { new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, new Short((short) 3) /*replication*/, - HdfsConstants.RS_6_3_POLICY_ID /*ec policy*/, + StripedFileTestUtil.getDefaultECPolicy().getId() /*ec policy*/, 1024L, HdfsConstants.WARM_STORAGE_POLICY_ID, STRIPED); fail("INodeFile construction should fail when both replication and " + "ECPolicy requested!"); @@ -147,7 +147,7 @@ public void testStripedLayoutRedundancy() { LOG.info("Expected exception: ", iae); } - final Byte ecPolicyID = HdfsConstants.RS_6_3_POLICY_ID; + final Byte ecPolicyID = StripedFileTestUtil.getDefaultECPolicy().getId(); try { new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, null /*replication*/, ecPolicyID, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java index 632bbf688e0..ca44c798010 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java @@ -292,7 +292,7 @@ public void testHAUtilClonesDelegationTokens() throws Exception { nn0.getNameNodeAddress().getPort())); nnAddrs.add(new InetSocketAddress("localhost", nn1.getNameNodeAddress().getPort())); - HAUtil.cloneDelegationTokenForLogicalUri(ugi, haUri, nnAddrs); + HAUtilClient.cloneDelegationTokenForLogicalUri(ugi, haUri, nnAddrs); Collection> tokens = ugi.getTokens(); assertEquals(3, tokens.size()); @@ -321,7 +321,7 @@ public void testHAUtilClonesDelegationTokens() throws Exception { } // reclone the tokens, and see if they match now - HAUtil.cloneDelegationTokenForLogicalUri(ugi, haUri, nnAddrs); + HAUtilClient.cloneDelegationTokenForLogicalUri(ugi, haUri, nnAddrs); for (InetSocketAddress addr : nnAddrs) { Text ipcDtService = SecurityUtil.buildTokenService(addr); Token token2 = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java index e29d51851ea..b40006be732 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java @@ -405,7 +405,8 @@ void invoke() throws Exception { FsPermission.getFileDefault(), client.getClientName(), new EnumSetWritable(createFlag), false, DataNodes, BlockSize, - new CryptoProtocolVersion[] {CryptoProtocolVersion.ENCRYPTION_ZONES}); + new CryptoProtocolVersion[] {CryptoProtocolVersion.ENCRYPTION_ZONES}, + null); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 0656249cfca..7182071a412 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -78,9 +78,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; @@ -124,9 +123,8 @@ public static void createOriginalFSImage() throws IOException { tempDir = Files.createTempDir(); MiniDFSCluster cluster = null; try { - final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.XOR_2_1_POLICY_ID); + final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); Configuration conf = new Configuration(); conf.setLong( @@ -412,8 +410,7 @@ public void endElement(String uri, String localName, String qName) Assert.assertEquals("INode '" + currentInodeName + "' has unexpected EC Policy!", Byte.parseByte(currentECPolicy), - ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.XOR_2_1_POLICY_ID).getId()); + SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); Assert.assertEquals("INode '" + currentInodeName + "' has unexpected replication!", currentRepl, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java index c1728ac97f1..5de63eb4ebd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -28,7 +29,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager; import static org.apache.hadoop.hdfs.util.StripedBlockUtil.*; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.junit.Before; import org.junit.Rule; @@ -81,7 +81,7 @@ public class TestStripedBlockUtil { // use hard coded policy - see HDFS-9816 private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemPolicies()[0]; + StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final short groupSize = (short) (dataBlocks + parityBlocks); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 46a8a56f1c1..60d90fb37b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -81,12 +81,12 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; @@ -520,8 +520,8 @@ public void testWebHdfsErasureCodingFiles() throws Exception { MiniDFSCluster cluster = null; final Configuration conf = WebHdfsTestUtil.createConf(); conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, - ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.XOR_2_1_POLICY_ID).getName()); + SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.XOR_2_1_POLICY_ID).getName()); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); cluster.waitActive(); @@ -532,8 +532,8 @@ public void testWebHdfsErasureCodingFiles() throws Exception { final Path ecDir = new Path("/ec"); dfs.mkdirs(ecDir); dfs.setErasureCodingPolicy(ecDir, - ErasureCodingPolicyManager.getPolicyByID( - HdfsConstants.XOR_2_1_POLICY_ID).getName()); + SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.XOR_2_1_POLICY_ID).getName()); final Path ecFile = new Path(ecDir, "ec-file.log"); DFSTestUtil.createFile(dfs, ecFile, 1024 * 10, (short) 1, 0xFEED); diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 77f0bb81fa3..cf17aea75a8 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -69,11 +69,13 @@ function mapredcmd_case historyserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer - if [ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]; then - # shellcheck disable=SC2034 + if [[ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]]; then HADOOP_HEAPSIZE_MAX="${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" fi HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER} + if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then + hadoop_add_param HADOOP_OPTS mapred.jobsummary.logger "-Dmapred.jobsummary.logger=${HADOOP_DAEMON_ROOT_LOGGER}" + fi ;; hsadmin) HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin @@ -112,9 +114,9 @@ else fi HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" -# shellcheck disable=SC2034 HADOOP_NEW_CONFIG=true if [[ -f "${HADOOP_LIBEXEC_DIR}/mapred-config.sh" ]]; then + # shellcheck source=./hadoop-mapreduce-project/bin/mapred-config.sh . "${HADOOP_LIBEXEC_DIR}/mapred-config.sh" else echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/mapred-config.sh." 2>&1 @@ -139,7 +141,7 @@ if hadoop_need_reexec mapred "${HADOOP_SUBCMD}"; then exit $? fi -hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" +hadoop_verify_user_perm "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@") @@ -159,55 +161,5 @@ fi hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" -if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then - HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}" - - hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" - - hadoop_verify_secure_prereq - hadoop_setup_secure_service - priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err" - priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" - daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" -else - daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" -fi - -if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then - # shellcheck disable=SC2034 - HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" - hadoop_add_param HADOOP_OPTS mapred.jobsummary.logger "-Dmapred.jobsummary.logger=${HADOOP_ROOT_LOGGER}" - # shellcheck disable=SC2034 - HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log" -fi - -hadoop_finalize - -if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then - if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then - hadoop_secure_daemon_handler \ - "${HADOOP_DAEMON_MODE}" \ - "${HADOOP_SUBCMD}" \ - "${HADOOP_CLASSNAME}" \ - "${daemon_pidfile}" \ - "${daemon_outfile}" \ - "${priv_pidfile}" \ - "${priv_outfile}" \ - "${priv_errfile}" \ - "${HADOOP_SUBCMD_ARGS[@]}" - else - hadoop_daemon_handler \ - "${HADOOP_DAEMON_MODE}" \ - "${HADOOP_SUBCMD}" \ - "${HADOOP_CLASSNAME}" \ - "${daemon_pidfile}" \ - "${daemon_outfile}" \ - "${HADOOP_SUBCMD_ARGS[@]}" - fi - exit $? -else - hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}" -fi +# everything is in globals at this point, so call the generic handler +hadoop_generic_java_subcmd_handler diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh index 68d3463de69..f370084fe53 100755 --- a/hadoop-mapreduce-project/bin/mapred-config.sh +++ b/hadoop-mapreduce-project/bin/mapred-config.sh @@ -59,6 +59,7 @@ if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_mc_this}")" >/dev/null && pwd -P) fi +# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh if [[ -n "${HADOOP_COMMON_HOME}" ]] && [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java index 085c96612d9..f1cad57dd41 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java @@ -23,6 +23,7 @@ import java.net.URISyntaxException; import java.util.Collection; import java.util.HashMap; +import java.util.LinkedList; import java.util.Map; import org.apache.commons.logging.Log; @@ -91,7 +92,7 @@ public void uploadResources(Job job, Path submitJobDir) throws IOException { submitJobDir = new Path(submitJobDir.toUri().getPath()); FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION); - FileSystem.mkdirs(jtFs, submitJobDir, mapredSysPerms); + mkdirs(jtFs, submitJobDir, mapredSysPerms); Collection files = conf.getStringCollection("tmpfiles"); Collection libjars = conf.getStringCollection("tmpjars"); @@ -116,18 +117,20 @@ public void uploadResources(Job job, Path submitJobDir) throws IOException { job.getCredentials()); } - private void uploadFiles(Configuration conf, Collection files, + @VisibleForTesting + void uploadFiles(Configuration conf, Collection files, Path submitJobDir, FsPermission mapredSysPerms, short submitReplication) throws IOException { Path filesDir = JobSubmissionFiles.getJobDistCacheFiles(submitJobDir); if (!files.isEmpty()) { - FileSystem.mkdirs(jtFs, filesDir, mapredSysPerms); + mkdirs(jtFs, filesDir, mapredSysPerms); for (String tmpFile : files) { URI tmpURI = null; try { tmpURI = new URI(tmpFile); } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); + throw new IllegalArgumentException("Error parsing files argument." + + " Argument must be a valid URI: " + tmpFile, e); } Path tmp = new Path(tmpURI); Path newPath = copyRemoteFiles(filesDir, tmp, conf, submitReplication); @@ -136,50 +139,83 @@ private void uploadFiles(Configuration conf, Collection files, DistributedCache.addCacheFile(pathURI, conf); } catch (URISyntaxException ue) { // should not throw a uri exception - throw new IOException("Failed to create uri for " + tmpFile, ue); + throw new IOException( + "Failed to create a URI (URISyntaxException) for the remote path " + + newPath + ". This was based on the files parameter: " + + tmpFile, + ue); } } } } - private void uploadLibJars(Configuration conf, Collection libjars, + // Suppress warning for use of DistributedCache (it is everywhere). + @SuppressWarnings("deprecation") + @VisibleForTesting + void uploadLibJars(Configuration conf, Collection libjars, Path submitJobDir, FsPermission mapredSysPerms, short submitReplication) throws IOException { Path libjarsDir = JobSubmissionFiles.getJobDistCacheLibjars(submitJobDir); if (!libjars.isEmpty()) { - FileSystem.mkdirs(jtFs, libjarsDir, mapredSysPerms); + mkdirs(jtFs, libjarsDir, mapredSysPerms); + Collection libjarURIs = new LinkedList<>(); + boolean foundFragment = false; for (String tmpjars : libjars) { - Path tmp = new Path(tmpjars); + URI tmpURI = null; + try { + tmpURI = new URI(tmpjars); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Error parsing libjars argument." + + " Argument must be a valid URI: " + tmpjars, e); + } + Path tmp = new Path(tmpURI); Path newPath = copyRemoteFiles(libjarsDir, tmp, conf, submitReplication); - - // Add each file to the classpath - DistributedCache.addFileToClassPath( - new Path(newPath.toUri().getPath()), conf, jtFs, !useWildcard); + try { + URI pathURI = getPathURI(newPath, tmpURI.getFragment()); + if (!foundFragment) { + foundFragment = pathURI.getFragment() != null; + } + DistributedCache.addFileToClassPath(new Path(pathURI.getPath()), conf, + jtFs, false); + libjarURIs.add(pathURI); + } catch (URISyntaxException ue) { + // should not throw a uri exception + throw new IOException( + "Failed to create a URI (URISyntaxException) for the remote path " + + newPath + ". This was based on the libjar parameter: " + + tmpjars, + ue); + } } - if (useWildcard) { - // Add the whole directory to the cache + if (useWildcard && !foundFragment) { + // Add the whole directory to the cache using a wild card Path libJarsDirWildcard = jtFs.makeQualified(new Path(libjarsDir, DistributedCache.WILDCARD)); - DistributedCache.addCacheFile(libJarsDirWildcard.toUri(), conf); + } else { + for (URI uri : libjarURIs) { + DistributedCache.addCacheFile(uri, conf); + } } } } - private void uploadArchives(Configuration conf, Collection archives, + @VisibleForTesting + void uploadArchives(Configuration conf, Collection archives, Path submitJobDir, FsPermission mapredSysPerms, short submitReplication) throws IOException { Path archivesDir = JobSubmissionFiles.getJobDistCacheArchives(submitJobDir); if (!archives.isEmpty()) { - FileSystem.mkdirs(jtFs, archivesDir, mapredSysPerms); + mkdirs(jtFs, archivesDir, mapredSysPerms); for (String tmpArchives : archives) { URI tmpURI; try { tmpURI = new URI(tmpArchives); } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); + throw new IllegalArgumentException("Error parsing archives argument." + + " Argument must be a valid URI: " + tmpArchives, e); } Path tmp = new Path(tmpURI); Path newPath = @@ -189,13 +225,18 @@ private void uploadArchives(Configuration conf, Collection archives, DistributedCache.addCacheArchive(pathURI, conf); } catch (URISyntaxException ue) { // should not throw an uri excpetion - throw new IOException("Failed to create uri for " + tmpArchives, ue); + throw new IOException( + "Failed to create a URI (URISyntaxException) for the remote path" + + newPath + ". This was based on the archive parameter: " + + tmpArchives, + ue); } } } } - private void uploadJobJar(Job job, String jobJar, Path submitJobDir, + @VisibleForTesting + void uploadJobJar(Job job, String jobJar, Path submitJobDir, short submitReplication) throws IOException { if (jobJar != null) { // copy jar to JobTracker's fs // use jar name if job is not named. @@ -273,7 +314,8 @@ Path stringToPath(String s) { URI uri = new URI(s); return new Path(uri.getScheme(), uri.getAuthority(), uri.getPath()); } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); + throw new IllegalArgumentException( + "Error parsing argument." + " Argument must be a valid URI: " + s, e); } } @@ -380,9 +422,20 @@ FileStatus getFileStatus(Map statCache, return status; } + /** + * Create a new directory in the passed filesystem. This wrapper method exists + * so that it can be overridden/stubbed during testing. + */ + @VisibleForTesting + boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) + throws IOException { + return FileSystem.mkdirs(fs, dir, permission); + } + // copies a file to the jobtracker filesystem and returns the path where it // was copied to - private Path copyRemoteFiles(Path parentDir, Path originalPath, + @VisibleForTesting + Path copyRemoteFiles(Path parentDir, Path originalPath, Configuration conf, short replication) throws IOException { // check if we do not need to copy the files // is jt using the same file system. @@ -400,10 +453,12 @@ private Path copyRemoteFiles(Path parentDir, Path originalPath, Path newPath = new Path(parentDir, originalPath.getName()); FileUtil.copy(remoteFs, originalPath, jtFs, newPath, false, conf); jtFs.setReplication(newPath, replication); + jtFs.makeQualified(newPath); return newPath; } - private void copyJar(Path originalJarPath, Path submitJarFile, + @VisibleForTesting + void copyJar(Path originalJarPath, Path submitJarFile, short replication) throws IOException { jtFs.copyFromLocalFile(originalJarPath, submitJarFile); jtFs.setReplication(submitJarFile, replication); @@ -427,7 +482,7 @@ private URI getPathURI(Path destPath, String fragment) URI pathURI = destPath.toUri(); if (pathURI.getFragment() == null) { if (fragment == null) { - pathURI = new URI(pathURI.toString() + "#" + destPath.getName()); + // no fragment, just return existing pathURI from destPath } else { pathURI = new URI(pathURI.toString() + "#" + fragment); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java index 8ba50a66b3c..20b7b7dcfba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java @@ -23,13 +23,19 @@ import java.io.IOException; import java.net.URI; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapred.JobConf; import org.junit.Assert; import org.junit.Test; @@ -69,13 +75,13 @@ public void testStringToPath() throws IOException { @Test public void testAllDefaults() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); runLimitsTest(b.build(), true, null); } @Test public void testNoLimitsWithResources() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); b.setNumOfDCArchives(1); b.setNumOfDCFiles(1); b.setNumOfTmpArchives(10); @@ -88,7 +94,7 @@ public void testNoLimitsWithResources() throws IOException { @Test public void testAtResourceLimit() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); b.setNumOfDCArchives(1); b.setNumOfDCFiles(1); b.setNumOfTmpArchives(1); @@ -101,7 +107,7 @@ public void testAtResourceLimit() throws IOException { @Test public void testOverResourceLimit() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); b.setNumOfDCArchives(1); b.setNumOfDCFiles(1); b.setNumOfTmpArchives(1); @@ -114,7 +120,7 @@ public void testOverResourceLimit() throws IOException { @Test public void testAtResourcesMBLimit() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); b.setNumOfDCArchives(1); b.setNumOfDCFiles(1); b.setNumOfTmpArchives(1); @@ -128,7 +134,7 @@ public void testAtResourcesMBLimit() throws IOException { @Test public void testOverResourcesMBLimit() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); b.setNumOfDCArchives(1); b.setNumOfDCFiles(2); b.setNumOfTmpArchives(1); @@ -142,7 +148,7 @@ public void testOverResourcesMBLimit() throws IOException { @Test public void testAtSingleResourceMBLimit() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); b.setNumOfDCArchives(1); b.setNumOfDCFiles(2); b.setNumOfTmpArchives(1); @@ -156,7 +162,7 @@ public void testAtSingleResourceMBLimit() throws IOException { @Test public void testOverSingleResourceMBLimit() throws IOException { - ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); + ResourceConf.Builder b = new ResourceConf.Builder(); b.setNumOfDCArchives(1); b.setNumOfDCFiles(2); b.setNumOfTmpArchives(1); @@ -168,20 +174,263 @@ public void testOverSingleResourceMBLimit() throws IOException { runLimitsTest(b.build(), false, ResourceViolation.SINGLE_RESOURCE_SIZE); } + private String destinationPathPrefix = "hdfs:///destinationPath/"; + private String[] expectedFilesNoFrags = + { destinationPathPrefix + "tmpFiles0.txt", + destinationPathPrefix + "tmpFiles1.txt", + destinationPathPrefix + "tmpFiles2.txt", + destinationPathPrefix + "tmpFiles3.txt", + destinationPathPrefix + "tmpFiles4.txt", + destinationPathPrefix + "tmpjars0.jar", + destinationPathPrefix + "tmpjars1.jar" }; + + private String[] expectedFilesWithFrags = + { destinationPathPrefix + "tmpFiles0.txt#tmpFilesfragment0.txt", + destinationPathPrefix + "tmpFiles1.txt#tmpFilesfragment1.txt", + destinationPathPrefix + "tmpFiles2.txt#tmpFilesfragment2.txt", + destinationPathPrefix + "tmpFiles3.txt#tmpFilesfragment3.txt", + destinationPathPrefix + "tmpFiles4.txt#tmpFilesfragment4.txt", + destinationPathPrefix + "tmpjars0.jar#tmpjarsfragment0.jar", + destinationPathPrefix + "tmpjars1.jar#tmpjarsfragment1.jar" }; + + // We use the local fs for the submitFS in the StubedUploader, so libjars + // should be replaced with a single path. + private String[] expectedFilesWithWildcard = + { destinationPathPrefix + "tmpFiles0.txt", + destinationPathPrefix + "tmpFiles1.txt", + destinationPathPrefix + "tmpFiles2.txt", + destinationPathPrefix + "tmpFiles3.txt", + destinationPathPrefix + "tmpFiles4.txt", + "file:///libjars-submit-dir/libjars/*" }; + + private String[] expectedArchivesNoFrags = + { destinationPathPrefix + "tmpArchives0.tgz", + destinationPathPrefix + "tmpArchives1.tgz" }; + + private String[] expectedArchivesWithFrags = + { destinationPathPrefix + "tmpArchives0.tgz#tmpArchivesfragment0.tgz", + destinationPathPrefix + "tmpArchives1.tgz#tmpArchivesfragment1.tgz" }; + + private String jobjarSubmitDir = "/jobjar-submit-dir"; + private String expectedJobJar = jobjarSubmitDir + "/job.jar"; + + @Test + public void testPathsWithNoFragNoSchemeRelative() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(2); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithScheme(false); + b.setPathsWithFrags(false); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesNoFrags, + expectedArchivesNoFrags, expectedJobJar); + } + + @Test + public void testPathsWithNoFragNoSchemeAbsolute() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(2); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithFrags(false); + b.setPathsWithScheme(false); + b.setAbsolutePaths(true); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesNoFrags, + expectedArchivesNoFrags, expectedJobJar); + } + + @Test + public void testPathsWithFragNoSchemeAbsolute() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(2); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithFrags(true); + b.setPathsWithScheme(false); + b.setAbsolutePaths(true); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesWithFrags, + expectedArchivesWithFrags, expectedJobJar); + } + + @Test + public void testPathsWithFragNoSchemeRelative() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(2); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithFrags(true); + b.setAbsolutePaths(false); + b.setPathsWithScheme(false); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesWithFrags, + expectedArchivesWithFrags, expectedJobJar); + } + + @Test + public void testPathsWithFragSchemeAbsolute() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(2); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithFrags(true); + b.setAbsolutePaths(true); + b.setPathsWithScheme(true); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesWithFrags, + expectedArchivesWithFrags, expectedJobJar); + } + + @Test + public void testPathsWithNoFragWithSchemeAbsolute() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(2); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithFrags(false); + b.setPathsWithScheme(true); + b.setAbsolutePaths(true); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesNoFrags, + expectedArchivesNoFrags, expectedJobJar); + } + + @Test + public void testPathsWithNoFragAndWildCard() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(4); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithFrags(false); + b.setPathsWithScheme(true); + b.setAbsolutePaths(true); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf, true); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesWithWildcard, + expectedArchivesNoFrags, expectedJobJar); + } + + @Test + public void testPathsWithFragsAndWildCard() throws IOException { + ResourceConf.Builder b = new ResourceConf.Builder(); + b.setNumOfTmpFiles(5); + b.setNumOfTmpLibJars(2); + b.setNumOfTmpArchives(2); + b.setJobJar(true); + b.setPathsWithFrags(true); + b.setPathsWithScheme(true); + b.setAbsolutePaths(true); + ResourceConf rConf = b.build(); + JobConf jConf = new JobConf(); + JobResourceUploader uploader = new StubedUploader(jConf, true); + + runTmpResourcePathTest(uploader, rConf, jConf, expectedFilesWithFrags, + expectedArchivesWithFrags, expectedJobJar); + } + + private void runTmpResourcePathTest(JobResourceUploader uploader, + ResourceConf rConf, JobConf jConf, String[] expectedFiles, + String[] expectedArchives, String expectedJobJar) throws IOException { + rConf.setupJobConf(jConf); + // We use a pre and post job object here because we need the post job object + // to get the new values set during uploadResources, but we need the pre job + // to set the job jar because JobResourceUploader#uploadJobJar uses the Job + // interface not the JobConf. The post job is automatically created in + // validateResourcePaths. + Job jobPre = Job.getInstance(jConf); + uploadResources(uploader, jConf, jobPre); + + validateResourcePaths(jConf, expectedFiles, expectedArchives, + expectedJobJar, jobPre); + } + + private void uploadResources(JobResourceUploader uploader, JobConf jConf, + Job job) throws IOException { + Collection files = jConf.getStringCollection("tmpfiles"); + Collection libjars = jConf.getStringCollection("tmpjars"); + Collection archives = jConf.getStringCollection("tmparchives"); + String jobJar = jConf.getJar(); + uploader.uploadFiles(jConf, files, new Path("/files-submit-dir"), null, + (short) 3); + uploader.uploadArchives(jConf, archives, new Path("/archives-submit-dir"), + null, (short) 3); + uploader.uploadLibJars(jConf, libjars, new Path("/libjars-submit-dir"), + null, (short) 3); + uploader.uploadJobJar(job, jobJar, new Path(jobjarSubmitDir), (short) 3); + } + + private void validateResourcePaths(JobConf jConf, String[] expectedFiles, + String[] expectedArchives, String expectedJobJar, Job preJob) + throws IOException { + Job j = Job.getInstance(jConf); + validateResourcePathsSub(j.getCacheFiles(), expectedFiles); + validateResourcePathsSub(j.getCacheArchives(), expectedArchives); + // We use a different job object here because the jobjar was set on a + // different job object + Assert.assertEquals("Job jar path is different than expected!", + expectedJobJar, preJob.getJar()); + } + + private void validateResourcePathsSub(URI[] actualURIs, + String[] expectedURIs) { + List actualList = Arrays.asList(actualURIs); + Set expectedSet = new HashSet<>(Arrays.asList(expectedURIs)); + if (actualList.size() != expectedSet.size()) { + Assert.fail("Expected list of resources (" + expectedSet.size() + + ") and actual list of resources (" + actualList.size() + + ") are different lengths!"); + } + + for (URI u : actualList) { + if (!expectedSet.contains(u.toString())) { + Assert.fail("Resource list contained unexpected path: " + u.toString()); + } + } + } + private enum ResourceViolation { NUMBER_OF_RESOURCES, TOTAL_RESOURCE_SIZE, SINGLE_RESOURCE_SIZE; } - private void runLimitsTest(ResourceLimitsConf rlConf, - boolean checkShouldSucceed, ResourceViolation violation) - throws IOException { + private void runLimitsTest(ResourceConf rlConf, boolean checkShouldSucceed, + ResourceViolation violation) throws IOException { if (!checkShouldSucceed && violation == null) { Assert.fail("Test is misconfigured. checkShouldSucceed is set to false" + " and a ResourceViolation is not specified."); } - JobConf conf = setupJobConf(rlConf); + JobConf conf = new JobConf(); + rlConf.setupJobConf(conf); JobResourceUploader uploader = new StubedUploader(conf); long configuredSizeOfResourceBytes = rlConf.sizeOfResource * 1024 * 1024; when(mockedStatus.getLen()).thenReturn(configuredSizeOfResourceBytes); @@ -230,43 +479,7 @@ private void runLimitsTest(ResourceLimitsConf rlConf, private final FileStatus mockedStatus = mock(FileStatus.class); - private JobConf setupJobConf(ResourceLimitsConf rlConf) { - JobConf conf = new JobConf(); - conf.setInt(MRJobConfig.MAX_RESOURCES, rlConf.maxResources); - conf.setLong(MRJobConfig.MAX_RESOURCES_MB, rlConf.maxResourcesMB); - conf.setLong(MRJobConfig.MAX_SINGLE_RESOURCE_MB, - rlConf.maxSingleResourceMB); - - conf.set("tmpfiles", - buildPathString("file:///tmpFiles", rlConf.numOfTmpFiles)); - conf.set("tmpjars", - buildPathString("file:///tmpjars", rlConf.numOfTmpLibJars)); - conf.set("tmparchives", - buildPathString("file:///tmpArchives", rlConf.numOfTmpArchives)); - conf.set(MRJobConfig.CACHE_ARCHIVES, - buildPathString("file:///cacheArchives", rlConf.numOfDCArchives)); - conf.set(MRJobConfig.CACHE_FILES, - buildPathString("file:///cacheFiles", rlConf.numOfDCFiles)); - if (rlConf.jobJar) { - conf.setJar("file:///jobjar.jar"); - } - return conf; - } - - private String buildPathString(String pathPrefix, int numOfPaths) { - if (numOfPaths < 1) { - return ""; - } else { - StringBuilder b = new StringBuilder(); - b.append(pathPrefix + 0); - for (int i = 1; i < numOfPaths; i++) { - b.append("," + pathPrefix + i); - } - return b.toString(); - } - } - - final static class ResourceLimitsConf { + private static class ResourceConf { private final int maxResources; private final long maxResourcesMB; private final long maxSingleResourceMB; @@ -277,14 +490,15 @@ final static class ResourceLimitsConf { private final int numOfDCFiles; private final int numOfDCArchives; private final long sizeOfResource; + private final boolean pathsWithFrags; + private final boolean pathsWithScheme; + private final boolean absolutePaths; - static final ResourceLimitsConf DEFAULT = new ResourceLimitsConf(); - - private ResourceLimitsConf() { + private ResourceConf() { this(new Builder()); } - private ResourceLimitsConf(Builder builder) { + private ResourceConf(Builder builder) { this.maxResources = builder.maxResources; this.maxResourcesMB = builder.maxResourcesMB; this.maxSingleResourceMB = builder.maxSingleResourceMB; @@ -295,6 +509,9 @@ private ResourceLimitsConf(Builder builder) { this.numOfDCFiles = builder.numOfDCFiles; this.numOfDCArchives = builder.numOfDCArchives; this.sizeOfResource = builder.sizeOfResource; + this.pathsWithFrags = builder.pathsWithFrags; + this.pathsWithScheme = builder.pathsWithScheme; + this.absolutePaths = builder.absolutePaths; } static class Builder { @@ -309,69 +526,176 @@ static class Builder { private int numOfDCFiles = 0; private int numOfDCArchives = 0; private long sizeOfResource = 0; + private boolean pathsWithFrags = false; + private boolean pathsWithScheme = false; + private boolean absolutePaths = true; - Builder() { + private Builder() { } - Builder setMaxResources(int max) { + private Builder setMaxResources(int max) { this.maxResources = max; return this; } - Builder setMaxResourcesMB(long max) { + private Builder setMaxResourcesMB(long max) { this.maxResourcesMB = max; return this; } - Builder setMaxSingleResourceMB(long max) { + private Builder setMaxSingleResourceMB(long max) { this.maxSingleResourceMB = max; return this; } - Builder setNumOfTmpFiles(int num) { + private Builder setNumOfTmpFiles(int num) { this.numOfTmpFiles = num; return this; } - Builder setNumOfTmpArchives(int num) { + private Builder setNumOfTmpArchives(int num) { this.numOfTmpArchives = num; return this; } - Builder setNumOfTmpLibJars(int num) { + private Builder setNumOfTmpLibJars(int num) { this.numOfTmpLibJars = num; return this; } - Builder setJobJar(boolean jar) { + private Builder setJobJar(boolean jar) { this.jobJar = jar; return this; } - Builder setNumOfDCFiles(int num) { + private Builder setNumOfDCFiles(int num) { this.numOfDCFiles = num; return this; } - Builder setNumOfDCArchives(int num) { + private Builder setNumOfDCArchives(int num) { this.numOfDCArchives = num; return this; } - Builder setSizeOfResource(long sizeMB) { + private Builder setSizeOfResource(long sizeMB) { this.sizeOfResource = sizeMB; return this; } - ResourceLimitsConf build() { - return new ResourceLimitsConf(this); + private Builder setPathsWithFrags(boolean fragments) { + this.pathsWithFrags = fragments; + return this; + } + + private Builder setPathsWithScheme(boolean scheme) { + this.pathsWithScheme = scheme; + return this; + } + + private Builder setAbsolutePaths(boolean absolute) { + this.absolutePaths = absolute; + return this; + } + + ResourceConf build() { + return new ResourceConf(this); + } + } + + private void setupJobConf(JobConf conf) { + conf.set("tmpfiles", + buildPathString("tmpFiles", this.numOfTmpFiles, ".txt")); + conf.set("tmpjars", + buildPathString("tmpjars", this.numOfTmpLibJars, ".jar")); + conf.set("tmparchives", + buildPathString("tmpArchives", this.numOfTmpArchives, ".tgz")); + conf.set(MRJobConfig.CACHE_ARCHIVES, buildDistributedCachePathString( + "cacheArchives", this.numOfDCArchives, ".tgz")); + conf.set(MRJobConfig.CACHE_FILES, buildDistributedCachePathString( + "cacheFiles", this.numOfDCFiles, ".txt")); + if (this.jobJar) { + String fragment = ""; + if (pathsWithFrags) { + fragment = "#jobjarfrag.jar"; + } + if (pathsWithScheme) { + conf.setJar("file:///jobjar.jar" + fragment); + } else { + if (absolutePaths) { + conf.setJar("/jobjar.jar" + fragment); + } else { + conf.setJar("jobjar.jar" + fragment); + } + } + } + conf.setInt(MRJobConfig.MAX_RESOURCES, this.maxResources); + conf.setLong(MRJobConfig.MAX_RESOURCES_MB, this.maxResourcesMB); + conf.setLong(MRJobConfig.MAX_SINGLE_RESOURCE_MB, + this.maxSingleResourceMB); + } + + // We always want absolute paths with a scheme in the DistributedCache, so + // we use a separate method to construct the path string. + private String buildDistributedCachePathString(String pathPrefix, + int numOfPaths, String extension) { + if (numOfPaths < 1) { + return ""; + } else { + StringBuilder b = new StringBuilder(); + b.append(buildPathStringSub(pathPrefix, "file:///" + pathPrefix, + extension, 0)); + for (int i = 1; i < numOfPaths; i++) { + b.append("," + buildPathStringSub(pathPrefix, "file:///" + pathPrefix, + extension, i)); + } + return b.toString(); + } + } + + private String buildPathString(String pathPrefix, int numOfPaths, + String extension) { + if (numOfPaths < 1) { + return ""; + } else { + StringBuilder b = new StringBuilder(); + String processedPath; + if (pathsWithScheme) { + processedPath = "file:///" + pathPrefix; + } else { + if (absolutePaths) { + processedPath = "/" + pathPrefix; + } else { + processedPath = pathPrefix; + } + } + b.append(buildPathStringSub(pathPrefix, processedPath, extension, 0)); + for (int i = 1; i < numOfPaths; i++) { + b.append("," + + buildPathStringSub(pathPrefix, processedPath, extension, i)); + } + return b.toString(); + } + } + + private String buildPathStringSub(String pathPrefix, String processedPath, + String extension, int num) { + if (pathsWithFrags) { + return processedPath + num + extension + "#" + pathPrefix + "fragment" + + num + extension; + } else { + return processedPath + num + extension; } } } - class StubedUploader extends JobResourceUploader { + private class StubedUploader extends JobResourceUploader { StubedUploader(JobConf conf) throws IOException { - super(FileSystem.getLocal(conf), false); + this(conf, false); + } + + StubedUploader(JobConf conf, boolean useWildcard) throws IOException { + super(FileSystem.getLocal(conf), useWildcard); } @Override @@ -379,5 +703,26 @@ FileStatus getFileStatus(Map statCache, Configuration job, Path p) throws IOException { return mockedStatus; } + + @Override + boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) + throws IOException { + // Do nothing. Stubbed out to avoid side effects. We don't actually need + // to create submit dirs. + return true; + } + + @Override + Path copyRemoteFiles(Path parentDir, Path originalPath, Configuration conf, + short replication) throws IOException { + return new Path(destinationPathPrefix + originalPath.getName()); + } + + @Override + void copyJar(Path originalJarPath, Path submitJarFile, short replication) + throws IOException { + // Do nothing. Stubbed out to avoid side effects. We don't actually need + // to copy the jar to the remote fs. + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index cbca3c8cdb1..eca51fb31c7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -203,7 +203,7 @@ public void testMRNewTimelineServiceEventHandling() throws Exception { MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestMRTimelineEventHandling.class.getSimpleName(), 1, true); + TestMRTimelineEventHandling.class.getSimpleName(), 1); cluster.init(conf); cluster.start(); LOG.info("A MiniMRYarnCluster get start."); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java index 45c7d1f79df..65b9dbd880b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java @@ -381,6 +381,9 @@ private MiniMRClientCluster createMiniClusterWithCapacityScheduler() // Expected queue names depending on Capacity Scheduler queue naming conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, CapacityScheduler.class); + // Default value is 90 - if you have low disk space, + // testNetworkedJob will fail + conf.set(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, "99"); return MiniMRClientClusterFactory.create(this.getClass(), 2, conf); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index 06e37dd0f0e..bcfc9635128 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -74,11 +74,7 @@ public MiniMRYarnCluster(String testName) { } public MiniMRYarnCluster(String testName, int noOfNMs) { - this(testName, noOfNMs, false); - } - @Deprecated - public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) { - super(testName, 1, noOfNMs, 4, 4, enableAHS); + super(testName, 1, noOfNMs, 4, 4); historyServerWrapper = new JobHistoryServerWrapper(); addService(historyServerWrapper); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 0d9e789da7e..1d692a53cf4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -104,6 +104,11 @@ hadoop-hdfs test + + org.apache.hadoop + hadoop-hdfs-client + test + com.google.inject.extensions guice-servlet diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c0f05c9047a..d4b2d83d5bb 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -74,6 +74,9 @@ 1.9.13 2.7.8 + + 1.7.25 + 1.0 @@ -106,7 +109,7 @@ for an open-ended enforcement --> [${javac.version},) - [3.0.2,) + [3.3.0,) -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError @@ -131,6 +134,7 @@ 1.0-alpha-8 900 1.11.86 + 2.3.4 ${project.version} @@ -545,11 +549,6 @@ commons-csv 1.0 - - xmlenc - xmlenc - 0.52 - org.apache.httpcomponents httpclient @@ -893,17 +892,17 @@ org.slf4j slf4j-api - 1.7.10 + ${slf4j.version} org.slf4j slf4j-log4j12 - 1.7.10 + ${slf4j.version} org.slf4j jul-to-slf4j - 1.7.10 + ${slf4j.version} org.eclipse.jdt @@ -1080,7 +1079,7 @@ org.hsqldb hsqldb - 2.0.0 + ${hsqldb.version} com.codahale.metrics diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index b17281be8f8..9eb55756cd3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1525,8 +1525,6 @@ public boolean mkdirs(Path path, FsPermission permission) throws IOException, * @throws IOException other IO problems * @throws AmazonClientException on failures inside the AWS SDK */ - // TODO: If we have created an empty file at /foo/bar and we then call - // mkdirs for /foo/bar/baz/roo what happens to the empty file /foo/bar/? private boolean innerMkdirs(Path f, FsPermission permission) throws IOException, FileAlreadyExistsException, AmazonClientException { LOG.debug("Making directory: {}", f); @@ -1561,6 +1559,7 @@ private boolean innerMkdirs(Path f, FsPermission permission) String key = pathToKey(f); createFakeDirectory(key); + deleteUnnecessaryFakeDirectories(f.getParent()); return true; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 6a11699ba5f..5ff9321dd54 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -339,15 +339,9 @@ public static AWSCredentialProviderList createAWSCredentialProviderSet( credentials.add(new BasicAWSCredentialsProvider( creds.getUser(), creds.getPassword())); credentials.add(new EnvironmentVariableCredentialsProvider()); - credentials.add( - SharedInstanceProfileCredentialsProvider.getInstance()); + credentials.add(InstanceProfileCredentialsProvider.getInstance()); } else { for (Class aClass : awsClasses) { - if (aClass == InstanceProfileCredentialsProvider.class) { - LOG.debug("Found {}, but will use {} instead.", aClass.getName(), - SharedInstanceProfileCredentialsProvider.class.getName()); - aClass = SharedInstanceProfileCredentialsProvider.class; - } credentials.add(createAWSCredentialProvider(conf, aClass)); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java deleted file mode 100644 index cbc07873f08..00000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a; - -import com.amazonaws.auth.InstanceProfileCredentialsProvider; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * A subclass of {@link InstanceProfileCredentialsProvider} that enforces - * instantiation of only a single instance. - * This credential provider calls the EC2 instance metadata service to obtain - * credentials. For highly multi-threaded applications, it's possible that - * multiple instances call the service simultaneously and overwhelm it with - * load. The service handles this by throttling the client with an HTTP 429 - * response or forcibly terminating the connection. Forcing use of a single - * instance reduces load on the metadata service by allowing all threads to - * share the credentials. The base class is thread-safe, and there is nothing - * that varies in the credentials across different instances of - * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton - * instance is safe. - * - * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After - * Hadoop upgrades to that version or higher, it's likely that we can remove - * this class. - */ -@InterfaceAudience.Private -@InterfaceStability.Stable -public final class SharedInstanceProfileCredentialsProvider - extends InstanceProfileCredentialsProvider { - - private static final SharedInstanceProfileCredentialsProvider INSTANCE = - new SharedInstanceProfileCredentialsProvider(); - - /** - * Returns the singleton instance. - * - * @return singleton instance - */ - public static SharedInstanceProfileCredentialsProvider getInstance() { - return INSTANCE; - } - - /** - * Default constructor, defined explicitly as private to enforce singleton. - */ - private SharedInstanceProfileCredentialsProvider() { - super(); - } -} diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 18c0cebdcba..a3a0bb1cf9d 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -328,13 +328,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. configuration of AWS access key ID and secret access key in environment variables named AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider: - a shared instance of - com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS - SDK, which supports use of instance profile credentials if running - in an EC2 VM. Using this shared instance potentially reduces load - on the EC2 instance metadata service for multi-threaded - applications. + 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use + of instance profile credentials if running in an EC2 VM. @@ -407,13 +402,12 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to obtain an AWS login from a different source in the system, including environment variables, JVM properties and configuration files. -There are four AWS Credential Providers inside the `hadoop-aws` JAR: +There are three AWS Credential Providers inside the `hadoop-aws` JAR: | classname | description | |-----------|-------------| | `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials | | `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials | -| `org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`| Shared instance of EC2 Metadata Credentials, which can reduce load on the EC2 instance metadata service. (See below.) | | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login | There are also many in the Amazon SDKs, in particular two which are automatically @@ -425,24 +419,13 @@ set up in the authentication chain: | `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables | -*EC2 Metadata Credentials with `SharedInstanceProfileCredentialsProvider`* +*EC2 Metadata Credentials with `InstanceProfileCredentialsProvider`* Applications running in EC2 may associate an IAM role with the VM and query the [EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for credentials to access S3. Within the AWS SDK, this functionality is -provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded -applications may trigger a high volume of calls to the instance metadata service -and trigger throttling: either an HTTP 429 response or a forcible close of the -connection. - -To mitigate against this problem, `hadoop-aws` ships with a variant of -`InstanceProfileCredentialsProvider` called -`SharedInstanceProfileCredentialsProvider`. Using this ensures that all -instances of S3A reuse the same instance profile credentials instead of issuing -a large volume of redundant metadata service calls. If -`fs.s3a.aws.credentials.provider` refers to -`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses -`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead. +provided by `InstanceProfileCredentialsProvider`, which internally enforces a +singleton instance in order to prevent throttling problem. *Session Credentials with `TemporaryAWSCredentialsProvider`* @@ -542,7 +525,7 @@ This means that the default S3A authentication chain can be defined as org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, com.amazonaws.auth.EnvironmentVariableCredentialsProvider, - org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider + com.amazonaws.auth.InstanceProfileCredentialsProvider @@ -929,7 +912,7 @@ role information available when deployed in Amazon EC2. ```xml fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider + com.amazonaws.auth.InstanceProfileCredentialsProvider ``` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 33740c8ab93..82a8b841caa 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -114,7 +114,7 @@ public void testDefaultChain() throws Exception { Arrays.asList( BasicAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class, - SharedInstanceProfileCredentialsProvider.class); + InstanceProfileCredentialsProvider.class); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2), @@ -128,7 +128,7 @@ public void testConfiguredChain() throws Exception { List> expectedClasses = Arrays.asList( EnvironmentVariableCredentialsProvider.class, - SharedInstanceProfileCredentialsProvider.class, + InstanceProfileCredentialsProvider.class, AnonymousAWSCredentialsProvider.class); conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses)); AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet( diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index d3a5565e6d5..31df22254c2 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -40,6 +40,8 @@ public final class AdlConfKeys { "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = "fs.adl.oauth2.access.token.provider.type"; + public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT = + TokenProviderType.ClientCredential; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 0b860b36592..e63f1152569 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -243,7 +243,8 @@ private AccessTokenProvider getAccessTokenProvider(Configuration config) Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders( config, AdlFileSystem.class); TokenProviderType type = conf.getEnum( - AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT); switch (type) { case RefreshToken: diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java index 55c8f812c25..d843d5556b3 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; import com.squareup.okhttp.mockwebserver.MockWebServer; @@ -84,6 +86,7 @@ public void preTestSetup() throws IOException, URISyntaxException { // Responses are returned in the same order that they are enqueued. fs = new TestableAdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 3867e74fa77..36498c6696c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -101,6 +101,7 @@ public void testClientCredTokenProvider() public void testCustomCredTokenProvider() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); @@ -115,6 +116,7 @@ public void testCustomCredTokenProvider() public void testInvalidProviderConfigurationForType() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); try { @@ -136,6 +138,7 @@ public void testInvalidProviderConfigurationForClassPath() Configuration conf = new Configuration(); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "wrong.classpath.CustomMockTokenProvider"); try { diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java index c594c6540b4..737534cf010 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java @@ -38,6 +38,8 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * Test access token provider behaviour with custom token provider and for token @@ -89,6 +91,8 @@ public static Collection testDataForTokenProvider() { */ public void init() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, typeOfTokenProviderClass.getName()); fileSystems = new TestableAdlFileSystem[fsObjectCount]; diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java index 908f8b8919e..334c372e7b1 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java @@ -29,6 +29,8 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * This class verifies path conversion to SDK. @@ -39,6 +41,8 @@ public class TestRelativePathFormation { public void testToRelativePath() throws URISyntaxException, IOException { AdlFileSystem fs = new AdlFileSystem(); Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "org.apache.hadoop.fs.adl.common.CustomMockTokenProvider"); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 5469944f9b3..e06522b8d44 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -1426,13 +1426,20 @@ NativeFileSystemStore getStoreInterface() { return store; } - private void performAuthCheck(String path, String accessType, - String operation) throws WasbAuthorizationException, IOException { + /** + * @param requestingAccessForPath - The path to the ancestor/parent/subtree/file that needs to be + * checked before granting access to originalPath + * @param accessType - The type of access READ/WRITE being requested + * @param operation - A string describing the operation being performed ("delete", "create" etc.). + * @param originalPath - The originalPath that was being accessed + */ + private void performAuthCheck(String requestingAccessForPath, WasbAuthorizationOperations accessType, + String operation, String originalPath) throws WasbAuthorizationException, IOException { if (azureAuthorization && this.authorizer != null && - !this.authorizer.authorize(path, accessType)) { + !this.authorizer.authorize(requestingAccessForPath, accessType.toString())) { throw new WasbAuthorizationException(operation - + " operation for Path : " + path + " not allowed"); + + " operation for Path : " + originalPath + " not allowed"); } } @@ -1459,8 +1466,7 @@ public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.WRITE.toString(), "append"); + performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "append", absolutePath.toString()); String key = pathToKey(absolutePath); FileMetadata meta = null; @@ -1663,9 +1669,9 @@ private FSDataOutputStream create(Path f, FsPermission permission, } Path absolutePath = makeAbsolute(f); + Path ancestor = getAncestor(absolutePath); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.WRITE.toString(), "create"); + performAuthCheck(ancestor.toString(), WasbAuthorizationOperations.WRITE, "create", absolutePath.toString()); String key = pathToKey(absolutePath); @@ -1678,6 +1684,9 @@ private FSDataOutputStream create(Path f, FsPermission permission, if (!overwrite) { throw new FileAlreadyExistsException("File already exists:" + f); } + else { + performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "create", absolutePath.toString()); + } } Path parentFolder = absolutePath.getParent(); @@ -1768,7 +1777,7 @@ public boolean delete(Path f, boolean recursive) throws IOException { /** * Delete the specified file or folder. The parameter - * skipParentFolderLastModifidedTimeUpdate + * skipParentFolderLastModifiedTimeUpdate * is used in the case of atomic folder rename redo. In that case, there is * a lease on the parent folder, so (without reworking the code) modifying * the parent folder update time will fail because of a conflict with the @@ -1778,20 +1787,20 @@ public boolean delete(Path f, boolean recursive) throws IOException { * * @param f file path to be deleted. * @param recursive specify deleting recursively or not. - * @param skipParentFolderLastModifidedTimeUpdate If true, don't update the folder last + * @param skipParentFolderLastModifiedTimeUpdate If true, don't update the folder last * modified time. * @return true if and only if the file is deleted * @throws IOException Thrown when fail to delete file or directory. */ public boolean delete(Path f, boolean recursive, - boolean skipParentFolderLastModifidedTimeUpdate) throws IOException { + boolean skipParentFolderLastModifiedTimeUpdate) throws IOException { LOG.debug("Deleting file: {}", f.toString()); Path absolutePath = makeAbsolute(f); + Path parentPath = absolutePath.getParent(); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "delete"); + performAuthCheck(parentPath.toString(), WasbAuthorizationOperations.WRITE, "delete", absolutePath.toString()); String key = pathToKey(absolutePath); @@ -1827,7 +1836,7 @@ public boolean delete(Path f, boolean recursive, // (e.g. the blob store only contains the blob a/b and there's no // corresponding directory blob a) and that would implicitly delete // the directory as well, which is not correct. - Path parentPath = absolutePath.getParent(); + if (parentPath.getParent() != null) {// Not root String parentKey = pathToKey(parentPath); @@ -1876,7 +1885,7 @@ public boolean delete(Path f, boolean recursive, store.storeEmptyFolder(parentKey, createPermissionStatus(FsPermission.getDefault())); } else { - if (!skipParentFolderLastModifidedTimeUpdate) { + if (!skipParentFolderLastModifiedTimeUpdate) { updateParentFolderLastModifiedTime(key); } } @@ -1903,7 +1912,6 @@ public boolean delete(Path f, boolean recursive, // The path specifies a folder. Recursively delete all entries under the // folder. LOG.debug("Directory Delete encountered: {}", f.toString()); - Path parentPath = absolutePath.getParent(); if (parentPath.getParent() != null) { String parentKey = pathToKey(parentPath); FileMetadata parentMetadata = null; @@ -1981,12 +1989,30 @@ public boolean delete(Path f, boolean recursive, final FileMetadata[] contents = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]); - if (!recursive && contents.length > 0) { + if (contents.length > 0) { + if (!recursive) { // The folder is non-empty and recursive delete was not specified. // Throw an exception indicating that a non-recursive delete was // specified for a non-empty folder. throw new IOException("Non-recursive delete of non-empty directory " + f.toString()); + } + else { + // Check write-permissions on sub-tree including current folder + // NOTE: Ideally the subtree needs read-write-execute access check. + // But we will simplify it to write-access check. + if (metaFile.isDir()) { // the absolute-path + performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "delete", + absolutePath.toString()); + } + for (FileMetadata meta : contents) { + if (meta.isDir()) { + Path subTreeDir = keyToPath(meta.getKey()); + performAuthCheck(subTreeDir.toString(), WasbAuthorizationOperations.WRITE, "delete", + absolutePath.toString()); + } + } + } } // Delete all files / folders in current directory stored as list in 'contents'. @@ -2014,7 +2040,7 @@ public boolean execute(FileMetadata file) throws IOException{ // Update parent directory last modified time Path parent = absolutePath.getParent(); if (parent != null && parent.getParent() != null) { // not root - if (!skipParentFolderLastModifidedTimeUpdate) { + if (!skipParentFolderLastModifiedTimeUpdate) { updateParentFolderLastModifiedTime(key); } } @@ -2064,8 +2090,8 @@ public FileStatus getFileStatus(Path f) throws FileNotFoundException, IOExceptio // Capture the absolute path and the path to key. Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "getFileStatus"); + performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "getFileStatus", + absolutePath.toString()); String key = pathToKey(absolutePath); if (key.length() == 0) { // root always exists @@ -2166,8 +2192,7 @@ public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "list"); + performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "liststatus", absolutePath.toString()); String key = pathToKey(absolutePath); Set status = new TreeSet(); @@ -2375,6 +2400,24 @@ PermissionStatus createPermissionStatus(FsPermission permission) permission); } + private Path getAncestor(Path f) throws IOException { + + for (Path current = f.getParent(), parent = current.getParent(); + parent != null; // Stop when you get to the root + current = parent, parent = current.getParent()) { + + String currentKey = pathToKey(current); + FileMetadata currentMetadata = store.retrieveMetadata(currentKey); + if (currentMetadata != null) { + Path ancestor = keyToPath(currentMetadata.getKey()); + LOG.debug("Found ancestor {}, for path: {}", ancestor.toString(), f.toString()); + return ancestor; + } + } + + return new Path("/"); + } + @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { return mkdirs(f, permission, false); @@ -2391,9 +2434,9 @@ public boolean mkdirs(Path f, FsPermission permission, boolean noUmask) throws I } Path absolutePath = makeAbsolute(f); + Path ancestor = getAncestor(absolutePath); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "mkdirs"); + performAuthCheck(ancestor.toString(), WasbAuthorizationOperations.WRITE, "mkdirs", absolutePath.toString()); PermissionStatus permissionStatus = null; if(noUmask) { @@ -2449,8 +2492,7 @@ public FSDataInputStream open(Path f, int bufferSize) throws FileNotFoundExcepti Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.READ.toString(), "read"); + performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "read", absolutePath.toString()); String key = pathToKey(absolutePath); FileMetadata meta = null; @@ -2508,12 +2550,18 @@ public boolean rename(Path src, Path dst) throws FileNotFoundException, IOExcept + " through WASB that has colons in the name"); } - Path absolutePath = makeAbsolute(src); + Path absoluteSrcPath = makeAbsolute(src); + Path srcParentFolder = absoluteSrcPath.getParent(); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "rename"); + if (srcParentFolder == null) { + // Cannot rename root of file system + return false; + } - String srcKey = pathToKey(absolutePath); + performAuthCheck(srcParentFolder.toString(), WasbAuthorizationOperations.WRITE, "rename", + absoluteSrcPath.toString()); + + String srcKey = pathToKey(absoluteSrcPath); if (srcKey.length() == 0) { // Cannot rename root of file system @@ -2521,8 +2569,13 @@ public boolean rename(Path src, Path dst) throws FileNotFoundException, IOExcept } // Figure out the final destination - Path absoluteDst = makeAbsolute(dst); - String dstKey = pathToKey(absoluteDst); + Path absoluteDstPath = makeAbsolute(dst); + Path dstParentFolder = absoluteDstPath.getParent(); + + performAuthCheck(dstParentFolder.toString(), WasbAuthorizationOperations.WRITE, "rename", + absoluteDstPath.toString()); + + String dstKey = pathToKey(absoluteDstPath); FileMetadata dstMetadata = null; try { dstMetadata = store.retrieveMetadata(dstKey); @@ -2530,14 +2583,14 @@ public boolean rename(Path src, Path dst) throws FileNotFoundException, IOExcept Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex); - // A BlobNotFound storage exception in only thrown from retrieveMetdata API when + // A BlobNotFound storage exception in only thrown from retrieveMetadata API when // there is a race condition. If there is another thread which deletes the destination // file or folder, then this thread calling rename should be able to continue with // rename gracefully. Hence the StorageException is swallowed here. if (innerException instanceof StorageException) { if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) { LOG.debug("BlobNotFound exception encountered for Destination key : {}. " - + "Swallowin the exception to handle race condition gracefully", dstKey); + + "Swallowing the exception to handle race condition gracefully", dstKey); } } else { throw ex; @@ -2558,7 +2611,7 @@ public boolean rename(Path src, Path dst) throws FileNotFoundException, IOExcept // Check that the parent directory exists. FileMetadata parentOfDestMetadata = null; try { - parentOfDestMetadata = store.retrieveMetadata(pathToKey(absoluteDst.getParent())); + parentOfDestMetadata = store.retrieveMetadata(pathToKey(absoluteDstPath.getParent())); } catch (IOException ex) { Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex); @@ -2816,9 +2869,6 @@ public Path getWorkingDirectory() { public void setPermission(Path p, FsPermission permission) throws FileNotFoundException, IOException { Path absolutePath = makeAbsolute(p); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "setPermission"); - String key = pathToKey(absolutePath); FileMetadata metadata = null; try { @@ -2858,9 +2908,6 @@ public void setOwner(Path p, String username, String groupname) throws IOException { Path absolutePath = makeAbsolute(p); - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "setOwner"); - String key = pathToKey(absolutePath); FileMetadata metadata = null; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java index e22a3a28318..4effb0d5811 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java @@ -132,7 +132,14 @@ public void init(Configuration conf) @Override public boolean authorize(String wasbAbsolutePath, String accessType) throws WasbAuthorizationException, IOException { + try { + + /* Make an exception for the internal -RenamePending files */ + if (wasbAbsolutePath.endsWith(NativeAzureFileSystem.FolderRenamePending.SUFFIX)) { + return true; + } + URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, @@ -203,7 +210,7 @@ public String run() throws Exception { return authorizerResponse.getAuthorizationResult(); } else { throw new WasbAuthorizationException("Remote authorization" - + " serivce encountered an error " + + " service encountered an error " + authorizerResponse.getResponseMessage()); } } catch (URISyntaxException | WasbRemoteCallException @@ -220,7 +227,7 @@ public String run() throws Exception { * response in the following JSON format * { * "responseCode" : 0 or non-zero , - * "responseMessage" : relavant message of failure + * "responseMessage" : relevant message of failure * "authorizationResult" : authorization result * true - if auhorization allowed * false - otherwise. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java index 41ca2b32267..7c63d4b8e3c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java @@ -34,8 +34,6 @@ public String toString() { return "read"; case WRITE: return "write"; - case EXECUTE: - return "execute"; default: throw new IllegalArgumentException( "Invalid Authorization Operation"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java index af5a537ce4d..445bfd8ea5f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; @@ -38,8 +39,11 @@ public void init(Configuration conf) { public void addAuthRule(String wasbAbsolutePath, String accessType, boolean access) { - AuthorizationComponent component = - new AuthorizationComponent(wasbAbsolutePath, accessType); + + AuthorizationComponent component = wasbAbsolutePath.endsWith("*") + ? new AuthorizationComponent("^" + wasbAbsolutePath.replace("*", ".*"), accessType) + : new AuthorizationComponent(wasbAbsolutePath, accessType); + this.authRules.put(component, access); } @@ -47,12 +51,26 @@ public void addAuthRule(String wasbAbsolutePath, public boolean authorize(String wasbAbsolutePath, String accessType) throws WasbAuthorizationException { + if (wasbAbsolutePath.endsWith(NativeAzureFileSystem.FolderRenamePending.SUFFIX)) { + return true; + } + AuthorizationComponent component = new AuthorizationComponent(wasbAbsolutePath, accessType); if (authRules.containsKey(component)) { return authRules.get(component); } else { + // Regex-pattern match if we don't have a straight match + for (Map.Entry entry : authRules.entrySet()) { + AuthorizationComponent key = entry.getKey(); + String keyPath = key.getWasbAbsolutePath(); + String keyAccess = key.getAccessType(); + + if (keyPath.endsWith("*") && Pattern.matches(keyPath, wasbAbsolutePath) && keyAccess.equals(accessType)) { + return entry.getValue(); + } + } return false; } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java index a2bbeb1765d..4e496227af4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java @@ -28,11 +28,8 @@ import org.junit.Rule; import org.junit.Test; -import com.sun.tools.javac.util.Assert; import org.junit.rules.ExpectedException; -import java.io.Console; - import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; /** @@ -67,32 +64,154 @@ public void beforeMethod() { public ExpectedException expectedEx = ExpectedException.none(); /** - * Positive test to verify Create and delete access check + * Setup up permissions to allow a recursive delete for cleanup purposes. + */ + private void allowRecursiveDelete(NativeAzureFileSystem fs, MockWasbAuthorizerImpl authorizer, String path) { + + int index = path.lastIndexOf('/'); + String parent = (index == 0) ? "/" : path.substring(0, index); + + authorizer.init(null); + authorizer.addAuthRule(parent, WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule((path.endsWith("*") ? path : path+"*"), WasbAuthorizationOperations.WRITE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + } + + /** + * Positive test to verify Create access check + * The file is created directly under an existing folder. + * No intermediate folders need to be created. * @throws Throwable */ @Test - public void testCreateAccessCheckPositive() throws Throwable { + public void testCreateAccessWithoutCreateIntermediateFoldersCheckPositive() throws Throwable { AzureBlobStorageTestAccount testAccount = createTestAccount(); NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/testCreateAccessCheckPositive"); + Path parentDir = new Path("/"); Path testPath = new Path(parentDir, "test.dat"); MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); fs.updateWasbAuthorizer(authorizer); - fs.create(testPath); - ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); - fs.delete(parentDir, true); + try { + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + } + finally { + fs.delete(testPath, false); + } } /** - * Negative test to verify Create access check + * Positive test to verify Create access check + * The test tries to create a file whose parent is non-existent to ensure that + * the intermediate folders between ancestor and direct parent are being created + * when proper ranger policies are configured. + * @throws Throwable + */ + @Test + public void testCreateAccessWithCreateIntermediateFoldersCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/testCreateAccessCheckPositive/1/2/3"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + } + finally { + allowRecursiveDelete(fs, authorizer, "/testCreateAccessCheckPositive"); + fs.delete(new Path("/testCreateAccessCheckPositive"), true); + } + } + + + /** + * Negative test to verify that create fails when trying to overwrite an existing file + * without proper write permissions on the file being overwritten. + * @throws Throwable + */ + @Test // (expected=WasbAuthorizationException.class) + public void testCreateAccessWithOverwriteCheckNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("create operation for Path : /test.dat not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + boolean initialCreateSucceeded = false; + try { + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + initialCreateSucceeded = true; + fs.create(testPath, true); + } + finally { + ContractTestUtils.assertTrue(initialCreateSucceeded); + fs.delete(testPath, false); + } + } + + /** + * Positive test to verify that create succeeds when trying to overwrite an existing file + * when proper write permissions on the file being overwritten are provided. + * @throws Throwable + */ + @Test + public void testCreateAccessWithOverwriteCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + boolean initialCreateSucceeded = false; + try { + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + initialCreateSucceeded = true; + fs.create(testPath, true); + } + finally { + ContractTestUtils.assertTrue(initialCreateSucceeded); + fs.delete(testPath, false); + } + } + + /** + * Negative test to verify that Create fails when appropriate permissions are not provided. * @throws Throwable */ @@ -110,20 +229,21 @@ public void testCreateAccessCheckNegative() throws Throwable { MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(),WasbAuthorizationOperations.WRITE.toString(), false); - authorizer.addAuthRule(parentDir.toString(),WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false); fs.updateWasbAuthorizer(authorizer); try { fs.create(testPath); } finally { + /* Provide permissions to cleanup in case the file got created */ + allowRecursiveDelete(fs, authorizer, parentDir.toString()); fs.delete(parentDir, true); } } /** - * Positive test to verify Create and delete access check + * Positive test to verify listStatus access check * @throws Throwable */ @Test @@ -133,28 +253,27 @@ public void testListAccessCheckPositive() throws Throwable { NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentDir = new Path("/testListAccessCheckPositive"); - Path testPath = new Path(parentDir, "test.dat"); + Path intermediateFolders = new Path(parentDir, "1/2/3/"); + Path testPath = new Path(intermediateFolders, "test.dat"); MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); fs.updateWasbAuthorizer(authorizer); - fs.create(testPath); - ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); - try { + fs.create(testPath); fs.listStatus(testPath); } finally { + allowRecursiveDelete(fs, authorizer, parentDir.toString()); fs.delete(parentDir, true); } } /** - * Negative test to verify Create access check + * Negative test to verify listStatus access check * @throws Throwable */ @@ -162,7 +281,7 @@ public void testListAccessCheckPositive() throws Throwable { public void testListAccessCheckNegative() throws Throwable { expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("getFileStatus operation for Path : /testListAccessCheckNegative/test.dat not allowed"); + expectedEx.expectMessage("liststatus operation for Path : /testListAccessCheckNegative/test.dat not allowed"); AzureBlobStorageTestAccount testAccount = createTestAccount(); NativeAzureFileSystem fs = testAccount.getFileSystem(); @@ -172,18 +291,16 @@ public void testListAccessCheckNegative() throws Throwable { MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), false); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false); fs.updateWasbAuthorizer(authorizer); - fs.create(testPath); - ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); - try { + fs.create(testPath); fs.listStatus(testPath); } finally { + allowRecursiveDelete(fs, authorizer, parentDir.toString()); fs.delete(parentDir, true); } } @@ -199,25 +316,26 @@ public void testRenameAccessCheckPositive() throws Throwable { NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentDir = new Path("/testRenameAccessCheckPositive"); - Path testPath = new Path(parentDir, "test.dat"); - Path renamePath = new Path(parentDir, "test2.dat"); + Path srcPath = new Path(parentDir, "test1.dat"); + Path dstPath = new Path(parentDir, "test2.dat"); MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(renamePath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parentDir */ + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); /* for rename */ + authorizer.addAuthRule(srcPath.toString(), WasbAuthorizationOperations.READ.toString(), true); /* for exists */ + authorizer.addAuthRule(dstPath.toString(), WasbAuthorizationOperations.READ.toString(), true); /* for exists */ fs.updateWasbAuthorizer(authorizer); - fs.create(testPath); - ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", testPath); - try { - fs.rename(testPath, renamePath); - ContractTestUtils.assertPathExists(fs, "destPath does not exist", renamePath); + fs.create(srcPath); + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", srcPath); + fs.rename(srcPath, dstPath); + ContractTestUtils.assertPathExists(fs, "destPath does not exist", dstPath); + ContractTestUtils.assertPathDoesNotExist(fs, "sourcePath exists after rename!", srcPath); } finally { + allowRecursiveDelete(fs, authorizer, parentDir.toString()); fs.delete(parentDir, true); } } @@ -230,37 +348,112 @@ public void testRenameAccessCheckPositive() throws Throwable { public void testRenameAccessCheckNegative() throws Throwable { expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegative/test.dat not allowed"); + expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegative/test1.dat not allowed"); AzureBlobStorageTestAccount testAccount = createTestAccount(); NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentDir = new Path("/testRenameAccessCheckNegative"); - Path testPath = new Path(parentDir, "test.dat"); - Path renamePath = new Path(parentDir, "test2.dat"); + Path srcPath = new Path(parentDir, "test1.dat"); + Path dstPath = new Path(parentDir, "test2.dat"); MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - // set EXECUTE to true for initial assert right after creation. - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); - fs.updateWasbAuthorizer(authorizer); - - fs.create(testPath); - ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", testPath); - - // Set EXECUTE to false for actual rename-failure test - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), false); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */ + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false); + authorizer.addAuthRule(srcPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + authorizer.addAuthRule(dstPath.toString(), WasbAuthorizationOperations.READ.toString(), true); fs.updateWasbAuthorizer(authorizer); try { - fs.rename(testPath, renamePath); - ContractTestUtils.assertPathExists(fs, "destPath does not exist", renamePath); + fs.create(srcPath); + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", srcPath); + fs.rename(srcPath, dstPath); + ContractTestUtils.assertPathExists(fs, "destPath does not exist", dstPath); } finally { + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist after rename failure!", srcPath); + + allowRecursiveDelete(fs, authorizer, parentDir.toString()); fs.delete(parentDir, true); } } + /** + * Negative test to verify rename access check - the dstFolder disallows rename + * @throws Throwable + */ + @Test //(expected=WasbAuthorizationException.class) + public void testRenameAccessCheckNegativeOnDstFolder() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegativeDst/test2.dat not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + Path parentSrcDir = new Path("/testRenameAccessCheckNegativeSrc"); + Path srcPath = new Path(parentSrcDir, "test1.dat"); + Path parentDstDir = new Path("/testRenameAccessCheckNegativeDst"); + Path dstPath = new Path(parentDstDir, "test2.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */ + authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(parentDstDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false); + authorizer.addAuthRule(srcPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + authorizer.addAuthRule(dstPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.create(srcPath); + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", srcPath); + fs.rename(srcPath, dstPath); + ContractTestUtils.assertPathDoesNotExist(fs, "destPath does not exist", dstPath); + } finally { + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist after rename !", srcPath); + allowRecursiveDelete(fs, authorizer, parentSrcDir.toString()); + fs.delete(parentSrcDir, true); + } + } + + /** + * Positive test to verify rename access check - the dstFolder allows rename + * @throws Throwable + */ + @Test //(expected=WasbAuthorizationException.class) + public void testRenameAccessCheckPositiveOnDstFolder() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + Path parentSrcDir = new Path("/testRenameAccessCheckPositiveSrc"); + Path srcPath = new Path(parentSrcDir, "test1.dat"); + Path parentDstDir = new Path("/testRenameAccessCheckPositiveDst"); + Path dstPath = new Path(parentDstDir, "test2.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dirs */ + authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(parentDstDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(srcPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + authorizer.addAuthRule(dstPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.create(srcPath); + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", srcPath); + fs.mkdirs(parentDstDir); + fs.rename(srcPath, dstPath); + ContractTestUtils.assertPathDoesNotExist(fs, "sourcePath does not exist", srcPath); + ContractTestUtils.assertPathExists(fs, "destPath does not exist", dstPath); + } finally { + allowRecursiveDelete(fs, authorizer, parentSrcDir.toString()); + fs.delete(parentSrcDir, true); + + allowRecursiveDelete(fs, authorizer, parentDstDir.toString()); + fs.delete(parentDstDir, true); + } + } + /** * Positive test for read access check. * @throws Throwable @@ -275,27 +468,30 @@ public void testReadAccessCheckPositive() throws Throwable { MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); fs.updateWasbAuthorizer(authorizer); - FSDataOutputStream fso = fs.create(testPath); - String data = "Hello World"; - fso.writeBytes(data); - fso.close(); - ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); - FSDataInputStream inputStream = null; + FSDataOutputStream fso = null; + try { + fso = fs.create(testPath); + String data = "Hello World"; + fso.writeBytes(data); + fso.close(); + inputStream = fs.open(testPath); ContractTestUtils.verifyRead(inputStream, data.getBytes(), 0, data.length()); } finally { + if (fso != null) { + fso.close(); + } if(inputStream != null) { inputStream.close(); } + allowRecursiveDelete(fs, authorizer, parentDir.toString()); fs.delete(parentDir, true); } } @@ -318,27 +514,239 @@ public void testReadAccessCheckNegative() throws Throwable { MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false); - authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); fs.updateWasbAuthorizer(authorizer); - FSDataOutputStream fso = fs.create(testPath); - String data = "Hello World"; - fso.writeBytes(data); - fso.close(); - ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); - FSDataInputStream inputStream = null; + FSDataOutputStream fso = null; + try { + fso = fs.create(testPath); + String data = "Hello World"; + fso.writeBytes(data); + fso.close(); + inputStream = fs.open(testPath); ContractTestUtils.verifyRead(inputStream, data.getBytes(), 0, data.length()); } finally { + if (fso != null) { + fso.close(); + } if (inputStream != null) { inputStream.close(); } + allowRecursiveDelete(fs, authorizer, parentDir.toString()); fs.delete(parentDir, true); } } + + /** + * Positive test to verify file delete access check + * @throws Throwable + */ + @Test + public void testFileDeleteAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + try { + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + } + finally { + fs.delete(testPath, false); + ContractTestUtils.assertPathDoesNotExist(fs, "testPath exists after deletion!", testPath); + } + } + + /** + * Negative test to verify file delete access check + * @throws Throwable + */ + @Test //(expected=WasbAuthorizationException.class) + public void testFileDeleteAccessCheckNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("delete operation for Path : /test.dat not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + try { + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + + + /* Remove permissions for delete to force failure */ + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false); + fs.updateWasbAuthorizer(authorizer); + + fs.delete(testPath, false); + } + finally { + /* Restore permissions to force a successful delete */ + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + fs.delete(testPath, false); + ContractTestUtils.assertPathDoesNotExist(fs, "testPath exists after deletion!", testPath); + } + } + + /** + * Positive test to verify file delete access check, with intermediate folders + * Uses wildcard recursive permissions + * @throws Throwable + */ + @Test + public void testFileDeleteAccessWithIntermediateFoldersCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/testDeleteIntermediateFolder"); + Path testPath = new Path(parentDir, "1/2/test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); // for create and delete + authorizer.addAuthRule("/testDeleteIntermediateFolder*", + WasbAuthorizationOperations.WRITE.toString(), true); // for recursive delete + authorizer.addAuthRule("/*", WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + fs.delete(parentDir, true); + ContractTestUtils.assertPathDoesNotExist(fs, "testPath exists after deletion!", parentDir); + } + finally { + allowRecursiveDelete(fs, authorizer, parentDir.toString()); + fs.delete(parentDir, true); + } + } + + /** + * Positive test for getFileStatus + * @throws Throwable + */ + @Test + public void testGetFileStatusPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path testPath = new Path("/"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + ContractTestUtils.assertIsDirectory(fs, testPath); + } + + /** + * Negative test for getFileStatus + * @throws Throwable + */ + @Test //(expected=WasbAuthorizationException.class) + public void testGetFileStatusNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("getFileStatus operation for Path : / not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path testPath = new Path("/"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.READ.toString(), false); + fs.updateWasbAuthorizer(authorizer); + + ContractTestUtils.assertIsDirectory(fs, testPath); + } + + /** + * Positive test for mkdirs access check + * @throws Throwable + */ + @Test + public void testMkdirsCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path testPath = new Path("/testMkdirsAccessCheckPositive/1/2/3"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.mkdirs(testPath); + ContractTestUtils.assertIsDirectory(fs, testPath); + } + finally { + allowRecursiveDelete(fs, authorizer, "/testMkdirsAccessCheckPositive"); + fs.delete(new Path("/testMkdirsAccessCheckPositive"), true); + } + } + + /** + * Negative test for mkdirs access check + * @throws Throwable + */ + @Test //(expected=WasbAuthorizationException.class) + public void testMkdirsCheckNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("mkdirs operation for Path : /testMkdirsAccessCheckNegative/1/2/3 not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path testPath = new Path("/testMkdirsAccessCheckNegative/1/2/3"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.mkdirs(testPath); + ContractTestUtils.assertPathDoesNotExist(fs, "testPath was not created", testPath); + } + finally { + allowRecursiveDelete(fs, authorizer, "/testMkdirsAccessCheckNegative"); + fs.delete(new Path("/testMkdirsAccessCheckNegative"), true); + } + } } \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java index b13e5e90574..d7c40b9dc87 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java @@ -263,9 +263,9 @@ public void testFailureCodeJSONResponse() throws Throwable { private void setupExpectations() { expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("org.apache.hadoop.fs.azure.WasbRemoteCallException: " + - "http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%2Ftest.dat&" + - "operation_type=write&delegation_token:Encountered IOException while making remote call"); + expectedEx.expectMessage("org.apache.hadoop.fs.azure.WasbRemoteCallException: " + + "http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%2F&" + + "operation_type=write:Encountered IOException while making remote call"); } private void performop(HttpClient mockHttpClient) throws Throwable { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java index 0c9126cf0a0..a1a021b16ee 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java @@ -117,7 +117,8 @@ public void testMetricsOnMkdirList() throws Exception { // levels, and then 2 requests for checking/stamping the version of AS, // totaling 11. // Also, there's the initial 1 request for container check so total is 12. - base = assertWebResponsesInRange(base, 1, 12); + // The getAncestor call at the very beginning adds another 4 calls, totalling 16. + base = assertWebResponsesInRange(base, 1, 16); assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_DIRECTORIES_CREATED)); diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml index 06c38f7bcf0..13e0ac0a536 100644 --- a/hadoop-tools/hadoop-kafka/pom.xml +++ b/hadoop-tools/hadoop-kafka/pom.xml @@ -36,32 +36,6 @@ true - - - tests-off - - - src/test/resources/auth-keys.xml - - - - true - - - - tests-on - - - src/test/resources/auth-keys.xml - - - - false - - - - - diff --git a/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java b/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java index eff1afa328b..8479298d56d 100644 --- a/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java +++ b/hadoop-tools/hadoop-kafka/src/test/java/org/apache/hadoop/metrics2/impl/TestKafkaMetrics.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.impl; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.AbstractMetric; @@ -74,7 +74,7 @@ public String description() { @Override public String toString() { - return Objects.toStringHelper(this).add("name", name()) + return MoreObjects.toStringHelper(this).add("name", name()) .add("description", desc).toString(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index b64879d93c0..cf6457b0438 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -120,7 +120,6 @@ function yarncmd_case HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer' # Backwards compatibility if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then - # shellcheck disable=SC2034 HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}" fi ;; @@ -132,7 +131,6 @@ function yarncmd_case HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' # Backwards compatibility if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then - # shellcheck disable=SC2034 HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}" fi ;; @@ -155,7 +153,6 @@ function yarncmd_case HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' # Backwards compatibility if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then - # shellcheck disable=SC2034 HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}" fi ;; @@ -210,9 +207,9 @@ else fi HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" -# shellcheck disable=SC2034 HADOOP_NEW_CONFIG=true if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then + # shellcheck source=./hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" else echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 @@ -239,7 +236,7 @@ if hadoop_need_reexec yarn "${HADOOP_SUBCMD}"; then exit $? fi -hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" +hadoop_verify_user_perm "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@") @@ -256,7 +253,6 @@ fi # HADOOP_CLIENT_OPTS instead before we (potentially) add it # to the command line if [[ -n "${YARN_CLIENT_OPTS}" ]]; then - # shellcheck disable=SC2034 HADOOP_CLIENT_OPTS=${YARN_CLIENT_OPTS} fi @@ -269,55 +265,5 @@ fi hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" -if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then - HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}" - - hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" - - hadoop_verify_secure_prereq - hadoop_setup_secure_service - priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err" - priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" - daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" -else - daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" - daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid" -fi - -if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then - # shellcheck disable=SC2034 - HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" - # shellcheck disable=SC2034 - HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log" -fi - -hadoop_finalize - -if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then - if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then - hadoop_secure_daemon_handler \ - "${HADOOP_DAEMON_MODE}" \ - "${HADOOP_SUBCMD}" \ - "${HADOOP_CLASSNAME}" \ - "${daemon_pidfile}" \ - "${daemon_outfile}" \ - "${priv_pidfile}" \ - "${priv_outfile}" \ - "${priv_errfile}" \ - "${HADOOP_SUBCMD_ARGS[@]}" - else - hadoop_daemon_handler \ - "${HADOOP_DAEMON_MODE}" \ - "${HADOOP_SUBCMD}" \ - "${HADOOP_CLASSNAME}" \ - "${daemon_pidfile}" \ - "${daemon_outfile}" \ - "${HADOOP_SUBCMD_ARGS[@]}" - fi - exit $? -else - # shellcheck disable=SC2086 - hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}" -fi +# everything is in globals at this point, so call the generic handler +hadoop_generic_java_subcmd_handler diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh index 708d5685bf5..150706e4141 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh @@ -69,6 +69,7 @@ if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_yc_this}")" >/dev/null && pwd -P) fi +# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh if [[ -n "${HADOOP_COMMON_HOME}" ]] && [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 81cb8c6c453..fa4d2e3d321 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -32,6 +32,7 @@ import org.apache.hadoop.ha.ActiveStandbyElector; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.BasicDiskValidator; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -974,7 +975,7 @@ public static boolean isAclEnabled(Configuration conf) { /** Disk Validator. */ public static final String DISK_VALIDATOR = NM_PREFIX + "disk-validator"; - public static final String DEFAULT_DISK_VALIDATOR = "basic"; + public static final String DEFAULT_DISK_VALIDATOR = BasicDiskValidator.NAME; /** * Maximum size of contain's diagnostics to keep for relaunching container diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index a564b823fa9..b3db4e764c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -151,6 +151,11 @@ hadoop-hdfs test + + org.apache.hadoop + hadoop-hdfs-client + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index a8e91323a38..c185abca44e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -274,7 +274,7 @@ protected void startHACluster(int numOfNMs, boolean overrideClientRMService, conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster = new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2, - numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS, + numOfNMs, 1, 1, overrideClientRMService, overrideRTS, overrideApplicationMasterService); cluster.resetStartFailoverFlag(false); cluster.init(conf); @@ -304,10 +304,10 @@ public class MiniYARNClusterForHATesting extends MiniYARNCluster { public MiniYARNClusterForHATesting(String testName, int numResourceManagers, int numNodeManagers, int numLocalDirs, - int numLogDirs, boolean enableAHS, boolean overrideClientRMService, + int numLogDirs, boolean overrideClientRMService, boolean overrideRTS, boolean overrideApplicationMasterService) { super(testName, numResourceManagers, numNodeManagers, numLocalDirs, - numLogDirs, enableAHS); + numLogDirs); this.overrideClientRMService = overrideClientRMService; this.overrideRTS = overrideRTS; this.overrideApplicationMasterService = overrideApplicationMasterService; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 05993d5dd9b..37c859cf5b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -1345,18 +1345,18 @@ private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, Path path = new Path(appDir, LogAggregationUtils.getNodeString(nodeId) + System.currentTimeMillis()); - AggregatedLogFormat.LogWriter writer = - new AggregatedLogFormat.LogWriter(configuration, path, ugi); - writer.writeApplicationOwner(ugi.getUserName()); + try (AggregatedLogFormat.LogWriter writer = + new AggregatedLogFormat.LogWriter()) { + writer.initialize(configuration, path, ugi); + writer.writeApplicationOwner(ugi.getUserName()); - Map appAcls = - new HashMap(); - appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); - writer.writeApplicationACLs(appAcls); - writer.append(new AggregatedLogFormat.LogKey(containerId), - new AggregatedLogFormat.LogValue(rootLogDirs, containerId, - UserGroupInformation.getCurrentUser().getShortUserName())); - writer.close(); + Map appAcls = new HashMap<>(); + appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); + writer.writeApplicationACLs(appAcls); + writer.append(new AggregatedLogFormat.LogKey(containerId), + new AggregatedLogFormat.LogValue(rootLogDirs, containerId, + UserGroupInformation.getCurrentUser().getShortUserName())); + } } private static void uploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi, @@ -1365,23 +1365,23 @@ private static void uploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ug Path path = new Path(appDir, LogAggregationUtils.getNodeString(nodeId) + System.currentTimeMillis()); - AggregatedLogFormat.LogWriter writer = - new AggregatedLogFormat.LogWriter(configuration, path, ugi); - writer.writeApplicationOwner(ugi.getUserName()); + try (AggregatedLogFormat.LogWriter writer = + new AggregatedLogFormat.LogWriter()) { + writer.initialize(configuration, path, ugi); + writer.writeApplicationOwner(ugi.getUserName()); - Map appAcls = - new HashMap(); - appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); - writer.writeApplicationACLs(appAcls); - DataOutputStream out = writer.getWriter().prepareAppendKey(-1); - new AggregatedLogFormat.LogKey(containerId).write(out); - out.close(); - out = writer.getWriter().prepareAppendValue(-1); - new AggregatedLogFormat.LogValue(rootLogDirs, containerId, - UserGroupInformation.getCurrentUser().getShortUserName()).write(out, - new HashSet()); - out.close(); - writer.close(); + Map appAcls = new HashMap<>(); + appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); + writer.writeApplicationACLs(appAcls); + DataOutputStream out = writer.getWriter().prepareAppendKey(-1); + new AggregatedLogFormat.LogKey(containerId).write(out); + out.close(); + out = writer.getWriter().prepareAppendValue(-1); + new AggregatedLogFormat.LogValue(rootLogDirs, containerId, + UserGroupInformation.getCurrentUser().getShortUserName()).write(out, + new HashSet<>()); + out.close(); + } } private YarnClient createMockYarnClient(YarnApplicationState appState, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index a2f53309989..013c227c0a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -187,14 +187,14 @@ public void replaceLabelsOnNode( dummyNodeLabelsManager.init(conf); } - @Test(timeout=500) + @Test public void testRefreshQueues() throws Exception { String[] args = { "-refreshQueues" }; assertEquals(0, rmAdminCLI.run(args)); verify(admin).refreshQueues(any(RefreshQueuesRequest.class)); } - @Test(timeout=500) + @Test public void testRefreshUserToGroupsMappings() throws Exception { String[] args = { "-refreshUserToGroupsMappings" }; assertEquals(0, rmAdminCLI.run(args)); @@ -202,7 +202,7 @@ public void testRefreshUserToGroupsMappings() throws Exception { any(RefreshUserToGroupsMappingsRequest.class)); } - @Test(timeout=500) + @Test public void testRefreshSuperUserGroupsConfiguration() throws Exception { String[] args = { "-refreshSuperUserGroupsConfiguration" }; assertEquals(0, rmAdminCLI.run(args)); @@ -210,14 +210,14 @@ public void testRefreshSuperUserGroupsConfiguration() throws Exception { any(RefreshSuperUserGroupsConfigurationRequest.class)); } - @Test(timeout=500) + @Test public void testRefreshAdminAcls() throws Exception { String[] args = { "-refreshAdminAcls" }; assertEquals(0, rmAdminCLI.run(args)); verify(admin).refreshAdminAcls(any(RefreshAdminAclsRequest.class)); } - @Test(timeout = 5000) + @Test public void testRefreshClusterMaxPriority() throws Exception { String[] args = { "-refreshClusterMaxPriority" }; assertEquals(0, rmAdminCLI.run(args)); @@ -225,14 +225,14 @@ public void testRefreshClusterMaxPriority() throws Exception { any(RefreshClusterMaxPriorityRequest.class)); } - @Test(timeout=500) + @Test public void testRefreshServiceAcl() throws Exception { String[] args = { "-refreshServiceAcl" }; assertEquals(0, rmAdminCLI.run(args)); verify(admin).refreshServiceAcls(any(RefreshServiceAclsRequest.class)); } - @Test(timeout=500) + @Test public void testUpdateNodeResource() throws Exception { String nodeIdStr = "0.0.0.0:0"; int memSize = 2048; @@ -256,7 +256,7 @@ public void testUpdateNodeResource() throws Exception { resource); } - @Test(timeout=500) + @Test public void testUpdateNodeResourceWithInvalidValue() throws Exception { String nodeIdStr = "0.0.0.0:0"; int memSize = -2048; @@ -270,7 +270,7 @@ public void testUpdateNodeResourceWithInvalidValue() throws Exception { any(UpdateNodeResourceRequest.class)); } - @Test(timeout=500) + @Test public void testRefreshNodes() throws Exception { String[] args = { "-refreshNodes" }; assertEquals(0, rmAdminCLI.run(args)); @@ -373,7 +373,7 @@ public void testRefreshNodesGracefulInvalidArgs() throws Exception { assertEquals(-1, rmAdminCLI.run(invalidTrackingArgs)); } - @Test(timeout=500) + @Test public void testGetGroups() throws Exception { when(admin.getGroupsForUser(eq("admin"))).thenReturn( new String[] {"group1", "group2"}); @@ -395,7 +395,7 @@ public boolean matches(Object argument) { } } - @Test(timeout = 500) + @Test public void testTransitionToActive() throws Exception { String[] args = {"-transitionToActive", "rm1"}; @@ -414,7 +414,7 @@ public void testTransitionToActive() throws Exception { verify(haadmin, times(1)).getServiceStatus(); } - @Test(timeout = 500) + @Test public void testTransitionToStandby() throws Exception { String[] args = {"-transitionToStandby", "rm1"}; @@ -431,7 +431,7 @@ public void testTransitionToStandby() throws Exception { any(HAServiceProtocol.StateChangeRequestInfo.class)); } - @Test(timeout = 500) + @Test public void testGetServiceState() throws Exception { String[] args = {"-getServiceState", "rm1"}; @@ -464,7 +464,7 @@ public void testGetAllServiceState() throws Exception { rmAdminCLIWithHAEnabled.setOut(System.out); } - @Test(timeout = 500) + @Test public void testCheckHealth() throws Exception { String[] args = {"-checkHealth", "rm1"}; @@ -482,7 +482,7 @@ public void testCheckHealth() throws Exception { /** * Test printing of help messages */ - @Test(timeout=500) + @Test public void testHelp() throws Exception { PrintStream oldOutPrintStream = System.out; PrintStream oldErrPrintStream = System.err; @@ -624,7 +624,7 @@ public void testHelp() throws Exception { } } - @Test(timeout=500) + @Test public void testException() throws Exception { PrintStream oldErrPrintStream = System.err; ByteArrayOutputStream dataErr = new ByteArrayOutputStream(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index 1f76c34960b..f07a9d6f055 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -208,11 +208,24 @@ public void setLocalResources( final Map localResources) { if (localResources == null) return; + checkLocalResources(localResources); initLocalResources(); this.localResources.clear(); this.localResources.putAll(localResources); } + private void checkLocalResources(Map localResources) { + for (Map.Entry rsrcEntry : localResources + .entrySet()) { + if (rsrcEntry.getValue() == null + || rsrcEntry.getValue().getResource() == null) { + throw new NullPointerException( + "Null resource URL for local resource " + rsrcEntry.getKey() + " : " + + rsrcEntry.getValue()); + } + } + } + private void addLocalResourcesToProto() { maybeInitBuilder(); builder.clearLocalResources(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 1b46007ebb9..0aa318c8ce8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -446,14 +446,23 @@ public boolean shouldRetainLog() { * The writer that writes out the aggregated logs. */ @Private - public static class LogWriter { + public static class LogWriter implements AutoCloseable { - private final FSDataOutputStream fsDataOStream; - private final TFile.Writer writer; + private FSDataOutputStream fsDataOStream; + private TFile.Writer writer; private FileContext fc; - public LogWriter(final Configuration conf, final Path remoteAppLogFile, - UserGroupInformation userUgi) throws IOException { + /** + * Initialize the LogWriter. + * Must be called just after the instance is created. + * @param conf Configuration + * @param remoteAppLogFile remote log file path + * @param userUgi Ugi of the user + * @throws IOException Failed to initialize + */ + public void initialize(final Configuration conf, + final Path remoteAppLogFile, + UserGroupInformation userUgi) throws IOException { try { this.fsDataOStream = userUgi.doAs(new PrivilegedExceptionAction() { @@ -530,13 +539,17 @@ public void append(LogKey logKey, LogValue logValue) throws IOException { } } + @Override public void close() { try { - this.writer.close(); - } catch (IOException e) { + if (writer != null) { + writer.close(); + } + } catch (Exception e) { LOG.warn("Exception closing writer", e); + } finally { + IOUtils.closeStream(this.fsDataOStream); } - IOUtils.closeStream(fsDataOStream); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java index 0294ad1ccc8..8773d11e2c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java @@ -30,6 +30,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.junit.Assert; import org.junit.Test; @@ -66,4 +70,29 @@ public void testCLCPBImplNullEnv() throws IOException { clcProto.getEnvironment().get("testCLCPBImplNullEnv")); } + + /* + * This test validates the scenario in which the client sets a null value for + * local resource URL. + */ + @Test + public void testCLCPBImplNullResourceURL() throws IOException { + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + try { + LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class); + rsrc_alpha.setResource(null); + rsrc_alpha.setSize(-1); + rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION); + rsrc_alpha.setType(LocalResourceType.FILE); + rsrc_alpha.setTimestamp(System.currentTimeMillis()); + Map localResources = + new HashMap(); + localResources.put("null_url_resource", rsrc_alpha); + ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); + containerLaunchContext.setLocalResources(localResources); + Assert.fail("Setting an invalid local resource should be an error!"); + } catch (NullPointerException e) { + Assert.assertTrue(e.getMessage().contains("Null resource URL for local resource")); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index 8cbec1065bf..efbaa4c44c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -140,44 +140,44 @@ private void writeSrcFileAndALog(Path srcFilePath, String fileName, final long l final int ch = filler; UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - LogWriter logWriter = new LogWriter(new Configuration(), remoteAppLogFile, - ugi); + try (LogWriter logWriter = new LogWriter()) { + logWriter.initialize(new Configuration(), remoteAppLogFile, ugi); - LogKey logKey = new LogKey(testContainerId); - LogValue logValue = - spy(new LogValue(Collections.singletonList(srcFileRoot.toString()), - testContainerId, ugi.getShortUserName())); + LogKey logKey = new LogKey(testContainerId); + LogValue logValue = + spy(new LogValue(Collections.singletonList(srcFileRoot.toString()), + testContainerId, ugi.getShortUserName())); - final CountDownLatch latch = new CountDownLatch(1); + final CountDownLatch latch = new CountDownLatch(1); - Thread t = new Thread() { - public void run() { - try { - for(int i=0; i < length/3; i++) { + Thread t = new Thread() { + public void run() { + try { + for (int i = 0; i < length / 3; i++) { osw.write(ch); - } + } - latch.countDown(); + latch.countDown(); - for(int i=0; i < (2*length)/3; i++) { - osw.write(ch); + for (int i = 0; i < (2 * length) / 3; i++) { + osw.write(ch); + } + osw.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } - osw.close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); } - } - }; - t.start(); + }; + t.start(); - //Wait till the osw is partially written - //aggregation starts once the ows has completed 1/3rd of its work - latch.await(); + //Wait till the osw is partially written + //aggregation starts once the ows has completed 1/3rd of its work + latch.await(); - //Aggregate The Logs - logWriter.append(logKey, logValue); - logWriter.close(); + //Aggregate The Logs + logWriter.append(logKey, logValue); + } } @Test @@ -216,22 +216,23 @@ private void testReadAcontainerLog(boolean logUploadedTime) throws Exception { writeSrcFile(srcFilePath, "stdout", numChars); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi); + try (LogWriter logWriter = new LogWriter()) { + logWriter.initialize(conf, remoteAppLogFile, ugi); - LogKey logKey = new LogKey(testContainerId); - LogValue logValue = - new LogValue(Collections.singletonList(srcFileRoot.toString()), - testContainerId, ugi.getShortUserName()); + LogKey logKey = new LogKey(testContainerId); + LogValue logValue = + new LogValue(Collections.singletonList(srcFileRoot.toString()), + testContainerId, ugi.getShortUserName()); - // When we try to open FileInputStream for stderr, it will throw out an IOException. - // Skip the log aggregation for stderr. - LogValue spyLogValue = spy(logValue); - File errorFile = new File((new Path(srcFilePath, "stderr")).toString()); - doThrow(new IOException("Mock can not open FileInputStream")).when( - spyLogValue).secureOpenFile(errorFile); + // When we try to open FileInputStream for stderr, it will throw out an + // IOException. Skip the log aggregation for stderr. + LogValue spyLogValue = spy(logValue); + File errorFile = new File((new Path(srcFilePath, "stderr")).toString()); + doThrow(new IOException("Mock can not open FileInputStream")).when( + spyLogValue).secureOpenFile(errorFile); - logWriter.append(logKey, spyLogValue); - logWriter.close(); + logWriter.append(logKey, spyLogValue); + } // make sure permission are correct on the file FileStatus fsStatus = fs.getFileStatus(remoteAppLogFile); @@ -311,24 +312,24 @@ public void testContainerLogsFileAccess() throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi); + try (LogWriter logWriter = new LogWriter()) { + logWriter.initialize(conf, remoteAppLogFile, ugi); - LogKey logKey = new LogKey(testContainerId1); - String randomUser = "randomUser"; - LogValue logValue = - spy(new LogValue(Collections.singletonList(srcFileRoot.toString()), - testContainerId1, randomUser)); - - // It is trying simulate a situation where first log file is owned by - // different user (probably symlink) and second one by the user itself. - // The first file should not be aggregated. Because this log file has the invalid - // user name. - when(logValue.getUser()).thenReturn(randomUser).thenReturn( - ugi.getShortUserName()); - logWriter.append(logKey, logValue); + LogKey logKey = new LogKey(testContainerId1); + String randomUser = "randomUser"; + LogValue logValue = + spy(new LogValue(Collections.singletonList(srcFileRoot.toString()), + testContainerId1, randomUser)); + + // It is trying simulate a situation where first log file is owned by + // different user (probably symlink) and second one by the user itself. + // The first file should not be aggregated. Because this log file has + // the invalid user name. + when(logValue.getUser()).thenReturn(randomUser).thenReturn( + ugi.getShortUserName()); + logWriter.append(logKey, logValue); + } - logWriter.close(); - BufferedReader in = new BufferedReader(new FileReader(new File(remoteAppLogFile .toUri().getRawPath()))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java index 594f18644f3..1e71b3cdca8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java @@ -295,17 +295,20 @@ private void writeLog(Configuration configuration, String user) List rootLogDirs = Arrays.asList("target/logs/logs"); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter( - configuration, new Path(path), ugi); - writer.writeApplicationOwner(ugi.getUserName()); + try (AggregatedLogFormat.LogWriter writer = + new AggregatedLogFormat.LogWriter()) { + writer.initialize(configuration, new Path(path), ugi); + writer.writeApplicationOwner(ugi.getUserName()); - Map appAcls = new HashMap(); - appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); - writer.writeApplicationACLs(appAcls); + Map appAcls = new HashMap<>(); + appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); + writer.writeApplicationACLs(appAcls); - writer.append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"), - new AggregatedLogFormat.LogValue(rootLogDirs, containerId,UserGroupInformation.getCurrentUser().getShortUserName())); - writer.close(); + writer.append( + new AggregatedLogFormat.LogKey("container_0_0001_01_000001"), + new AggregatedLogFormat.LogValue(rootLogDirs, containerId, + UserGroupInformation.getCurrentUser().getShortUserName())); + } } private void writeLogs(String dirName) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java index c6841c90f7d..8b665e03f6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java @@ -110,13 +110,14 @@ private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, ContainerId containerId, Path appDir, FileSystem fs) throws IOException { Path path = new Path(appDir, LogAggregationUtils.getNodeString(nodeId)); - AggregatedLogFormat.LogWriter writer = - new AggregatedLogFormat.LogWriter(configuration, path, ugi); - writer.writeApplicationOwner(ugi.getUserName()); + try (AggregatedLogFormat.LogWriter writer = + new AggregatedLogFormat.LogWriter()) { + writer.initialize(configuration, path, ugi); + writer.writeApplicationOwner(ugi.getUserName()); - writer.append(new AggregatedLogFormat.LogKey(containerId), - new AggregatedLogFormat.LogValue(rootLogDirs, containerId, - ugi.getShortUserName())); - writer.close(); + writer.append(new AggregatedLogFormat.LogKey(containerId), + new AggregatedLogFormat.LogValue(rootLogDirs, containerId, + ugi.getShortUserName())); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java index 72c32e8b50e..ae2a4ef1ca4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java @@ -181,7 +181,8 @@ public DirectoryCollection(String[] dirs, conf = new YarnConfiguration(); try { diskValidator = DiskValidatorFactory.getInstance( - conf.get(YarnConfiguration.DISK_VALIDATOR)); + conf.get(YarnConfiguration.DISK_VALIDATOR, + YarnConfiguration.DEFAULT_DISK_VALIDATOR)); LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR + " is loaded."); } catch (Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 60bd00bc583..1cff53f2116 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -82,6 +82,24 @@ public class NodeManager extends CompositeService implements EventHandler { + /** + * Node manager return status codes. + */ + public enum NodeManagerStatus { + NO_ERROR(0), + EXCEPTION(1); + + private int exitCode; + + NodeManagerStatus(int exitCode) { + this.exitCode = exitCode; + } + + public int getExitCode() { + return exitCode; + } + } + /** * Priority of the NodeManager shutdown hook. */ @@ -421,7 +439,7 @@ public String getName() { return "NodeManager"; } - protected void shutDown() { + protected void shutDown(final int exitCode) { new Thread() { @Override public void run() { @@ -432,7 +450,7 @@ public void run() { } finally { if (shouldExitOnShutdownEvent && !ShutdownHookManager.get().isShutdownInProgress()) { - ExitUtil.terminate(-1); + ExitUtil.terminate(exitCode); } } } @@ -457,7 +475,7 @@ public void run() { .rebootNodeStatusUpdaterAndRegisterWithRM(); } catch (YarnRuntimeException e) { LOG.fatal("Error while rebooting NodeStatusUpdater.", e); - shutDown(); + shutDown(NodeManagerStatus.EXCEPTION.getExitCode()); } } }.start(); @@ -744,7 +762,7 @@ private void initAndStartNodeManager(Configuration conf, boolean hasToReboot) { public void handle(NodeManagerEvent event) { switch (event.getType()) { case SHUTDOWN: - shutDown(); + shutDown(NodeManagerStatus.NO_ERROR.getExitCode()); break; case RESYNC: resyncWithRM(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index f3fe8ccc6a7..d82c728780d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NodeId; @@ -229,7 +230,8 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec, this.metrics = metrics; rsrcLocalizationSrvc = - createResourceLocalizationService(exec, deletionContext, context); + createResourceLocalizationService(exec, deletionContext, context, + metrics); addService(rsrcLocalizationSrvc); containersLauncher = createContainersLauncher(context, exec); @@ -476,9 +478,10 @@ public ContainersMonitor getContainersMonitor() { } protected ResourceLocalizationService createResourceLocalizationService( - ContainerExecutor exec, DeletionService deletionContext, Context context) { + ContainerExecutor exec, DeletionService deletionContext, + Context nmContext, NodeManagerMetrics nmMetrics) { return new ResourceLocalizationService(this.dispatcher, exec, - deletionContext, dirsHandler, context); + deletionContext, dirsHandler, nmContext, nmMetrics); } protected SharedCacheUploadService createSharedCacheUploaderService() { @@ -996,6 +999,15 @@ protected void startContainerInternal( ContainerLaunchContext launchContext = request.getContainerLaunchContext(); + // Sanity check for local resources + for (Map.Entry rsrc : launchContext + .getLocalResources().entrySet()) { + if (rsrc.getValue() == null || rsrc.getValue().getResource() == null) { + throw new YarnException( + "Null resource URL for local resource " + rsrc.getKey() + " : " + rsrc.getValue()); + } + } + Credentials credentials = YarnServerSecurityUtils.parseCredentials(launchContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 959092bdc7a..6e798573c17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -127,7 +127,8 @@ public ContainerLocalizer(FileContext lfs, String user, String appId, this.recordFactory = recordFactory; this.conf = new YarnConfiguration(); this.diskValidator = DiskValidatorFactory.getInstance( - conf.get(YarnConfiguration.DISK_VALIDATOR)); + conf.get(YarnConfiguration.DISK_VALIDATOR, + YarnConfiguration.DEFAULT_DISK_VALIDATOR)); LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR + " is loaded."); this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 37473e3192c..28fb53ce458 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -131,6 +131,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext; +import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredLocalizationState; @@ -165,6 +166,8 @@ public class ResourceLocalizationService extends CompositeService private final ScheduledExecutorService cacheCleanup; private LocalizerTokenSecretManager secretManager; private NMStateStoreService stateStore; + @VisibleForTesting + final NodeManagerMetrics metrics; @VisibleForTesting LocalResourcesTracker publicRsrc; @@ -194,7 +197,8 @@ public class ResourceLocalizationService extends CompositeService public ResourceLocalizationService(Dispatcher dispatcher, ContainerExecutor exec, DeletionService delService, - LocalDirsHandlerService dirsHandler, Context context) { + LocalDirsHandlerService dirsHandler, Context context, + NodeManagerMetrics metrics) { super(ResourceLocalizationService.class.getName()); this.exec = exec; @@ -208,6 +212,7 @@ public ResourceLocalizationService(Dispatcher dispatcher, .build()); this.stateStore = context.getNMStateStore(); this.nmContext = context; + this.metrics = metrics; } FileContext getLocalFileContext(Configuration conf) { @@ -256,7 +261,8 @@ public void serviceInit(Configuration conf) throws Exception { } diskValidator = DiskValidatorFactory.getInstance( - conf.get(YarnConfiguration.DISK_VALIDATOR)); + conf.get(YarnConfiguration.DISK_VALIDATOR, + YarnConfiguration.DEFAULT_DISK_VALIDATOR)); LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR + " is loaded."); cacheTargetSize = @@ -530,6 +536,12 @@ LocalCacheCleanerStats handleCacheCleanup() { } else if (LOG.isInfoEnabled()) { LOG.info(stats.toString()); } + + // Update metrics + metrics.setCacheSizeBeforeClean(stats.getCacheSizeBeforeClean()); + metrics.setTotalBytesDeleted(stats.getTotalDelSize()); + metrics.setPrivateBytesDeleted(stats.getPrivateDelSize()); + metrics.setPublicBytesDeleted(stats.getPublicDelSize()); return stats; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index d70acc940e4..f465534b66b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -295,18 +295,18 @@ private void uploadLogsForContainers(boolean appFinished) { } } - LogWriter writer = null; + if (pendingContainerInThisCycle.isEmpty()) { + sendLogAggregationReport(true, "", appFinished); + return; + } + + logAggregationTimes++; String diagnosticMessage = ""; boolean logAggregationSucceedInThisCycle = true; - try { - if (pendingContainerInThisCycle.isEmpty()) { - return; - } - - logAggregationTimes++; - + try (LogWriter writer = createLogWriter()) { try { - writer = createLogWriter(); + writer.initialize(this.conf, this.remoteNodeTmpLogFileForApp, + this.userUgi); // Write ACLs once when the writer is created. writer.writeApplicationACLs(appAcls); writer.writeApplicationOwner(this.userUgi.getShortUserName()); @@ -351,11 +351,6 @@ private void uploadLogsForContainers(boolean appFinished) { cleanupOldLogTimes++; } - if (writer != null) { - writer.close(); - writer = null; - } - long currentTime = System.currentTimeMillis(); final Path renamedPath = this.rollingMonitorInterval <= 0 ? remoteNodeLogFileForApp : new Path( @@ -396,34 +391,37 @@ public Object run() throws Exception { logAggregationSucceedInThisCycle = false; } } finally { - LogAggregationStatus logAggregationStatus = - logAggregationSucceedInThisCycle - ? LogAggregationStatus.RUNNING - : LogAggregationStatus.RUNNING_WITH_FAILURE; - sendLogAggregationReport(logAggregationStatus, diagnosticMessage); - if (appFinished) { - // If the app is finished, one extra final report with log aggregation - // status SUCCEEDED/FAILED will be sent to RM to inform the RM - // that the log aggregation in this NM is completed. - LogAggregationStatus finalLogAggregationStatus = - renameTemporaryLogFileFailed || !logAggregationSucceedInThisCycle - ? LogAggregationStatus.FAILED - : LogAggregationStatus.SUCCEEDED; - sendLogAggregationReport(finalLogAggregationStatus, ""); - } - - if (writer != null) { - writer.close(); - } + sendLogAggregationReport(logAggregationSucceedInThisCycle, + diagnosticMessage, appFinished); } } - protected LogWriter createLogWriter() throws IOException { - return new LogWriter(this.conf, this.remoteNodeTmpLogFileForApp, - this.userUgi); + @VisibleForTesting + protected LogWriter createLogWriter() { + return new LogWriter(); } private void sendLogAggregationReport( + boolean logAggregationSucceedInThisCycle, String diagnosticMessage, + boolean appFinished) { + LogAggregationStatus logAggregationStatus = + logAggregationSucceedInThisCycle + ? LogAggregationStatus.RUNNING + : LogAggregationStatus.RUNNING_WITH_FAILURE; + sendLogAggregationReportInternal(logAggregationStatus, diagnosticMessage); + if (appFinished) { + // If the app is finished, one extra final report with log aggregation + // status SUCCEEDED/FAILED will be sent to RM to inform the RM + // that the log aggregation in this NM is completed. + LogAggregationStatus finalLogAggregationStatus = + renameTemporaryLogFileFailed || !logAggregationSucceedInThisCycle + ? LogAggregationStatus.FAILED + : LogAggregationStatus.SUCCEEDED; + sendLogAggregationReportInternal(finalLogAggregationStatus, ""); + } + } + + private void sendLogAggregationReportInternal( LogAggregationStatus logAggregationStatus, String diagnosticMessage) { LogAggregationReport report = Records.newRecord(LogAggregationReport.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index 291b488c66f..a59bb5c6098 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -69,6 +69,15 @@ public class NodeManagerMetrics { @Metric("# of running opportunistic containers") MutableGaugeInt runningOpportunisticContainers; + @Metric("Local cache size (public and private) before clean (Bytes)") + MutableGaugeLong cacheSizeBeforeClean; + @Metric("# of total bytes deleted from the public and private local cache") + MutableGaugeLong totalBytesDeleted; + @Metric("# of bytes deleted from the public local cache") + MutableGaugeLong publicBytesDeleted; + @Metric("# of bytes deleted from the private local cache") + MutableGaugeLong privateBytesDeleted; + // CHECKSTYLE:ON:VisibilityModifier private JvmMetrics jvmMetrics = null; @@ -215,6 +224,22 @@ public void setGoodLogDirsDiskUtilizationPerc( this.goodLogDirsDiskUtilizationPerc.set(goodLogDirsDiskUtilizationPerc); } + public void setCacheSizeBeforeClean(long cacheSizeBeforeClean) { + this.cacheSizeBeforeClean.set(cacheSizeBeforeClean); + } + + public void setTotalBytesDeleted(long totalBytesDeleted) { + this.totalBytesDeleted.set(totalBytesDeleted); + } + + public void setPublicBytesDeleted(long publicBytesDeleted) { + this.publicBytesDeleted.set(publicBytesDeleted); + } + + public void setPrivateBytesDeleted(long privateBytesDeleted) { + this.privateBytesDeleted.set(privateBytesDeleted); + } + public int getRunningContainers() { return containersRunning.value(); } @@ -275,4 +300,20 @@ public int getAllocatedOpportunisticVCores() { public int getRunningOpportunisticContainers() { return runningOpportunisticContainers.value(); } + + public long getCacheSizeBeforeClean() { + return this.cacheSizeBeforeClean.value(); + } + + public long getTotalBytesDeleted() { + return this.totalBytesDeleted.value(); + } + + public long getPublicBytesDeleted() { + return this.publicBytesDeleted.value(); + } + + public long getPrivateBytesDeleted() { + return this.privateBytesDeleted.value(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java index 740ed19dc3e..8dc17a6f439 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java @@ -73,9 +73,10 @@ public DummyContainerManager(Context context, ContainerExecutor exec, @Override @SuppressWarnings("unchecked") protected ResourceLocalizationService createResourceLocalizationService( - ContainerExecutor exec, DeletionService deletionContext, Context context) { + ContainerExecutor exec, DeletionService deletionContext, Context context, + NodeManagerMetrics metrics) { return new ResourceLocalizationService(super.dispatcher, exec, - deletionContext, super.dirsHandler, context) { + deletionContext, super.dirsHandler, context, metrics) { @Override public void handle(LocalizationEvent event) { switch (event.getType()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java index 82218279e32..028db6aee9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager; import org.junit.After; +import org.junit.Assume; public class TestContainerManagerWithLCE extends TestContainerManager { @@ -51,11 +52,8 @@ public TestContainerManagerWithLCE() throws UnsupportedFileSystemException { @Override public void setup() throws IOException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } + Assume.assumeTrue("LCE binary path is not passed. Not running the test", + shouldRunTest()); super.setup(); localFS.setPermission(new Path(localDir.getCanonicalPath()), new FsPermission( @@ -73,297 +71,6 @@ public void tearDown() throws IOException, InterruptedException { } } - @Override - public void testContainerSetup() throws Exception, InterruptedException, - YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerSetup"); - super.testContainerSetup(); - } - - @Override - public void testContainerManagerInitialization() throws IOException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerManagerInitialization"); - super.testContainerManagerInitialization(); - } - - @Override - public void testContainerLaunchAndStop() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerLaunchAndStop"); - super.testContainerLaunchAndStop(); - } - - @Override - public void testContainerLaunchAndExitSuccess() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerLaunchAndExitSuccess"); - super.testContainerLaunchAndExitSuccess(); - } - - @Override - public void testContainerLaunchAndExitFailure() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerLaunchAndExitFailure"); - super.testContainerLaunchAndExitFailure(); - } - - @Override - public void testLocalingResourceWhileContainerRunning() - throws Exception { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - super.testLocalingResourceWhileContainerRunning(); - } - - @Override - public void testLocalFilesCleanup() throws InterruptedException, - IOException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testLocalFilesCleanup"); - super.testLocalFilesCleanup(); - } - - @Override - public void testContainerLaunchFromPreviousRM() throws InterruptedException, - IOException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerLaunchFromPreviousRM"); - super.testContainerLaunchFromPreviousRM(); - } - - @Override - public void testMultipleContainersLaunch() throws Exception { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerLaunchFromPreviousRM"); - super.testMultipleContainersLaunch(); - } - - @Override - public void testMultipleContainersStopAndGetStatus() throws Exception { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerLaunchFromPreviousRM"); - super.testMultipleContainersStopAndGetStatus(); - } - - @Override - public void testUnauthorizedRequests() throws IOException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testUnauthorizedRequests"); - super.testUnauthorizedRequests(); - } - - @Override - public void testStartContainerFailureWithUnknownAuxService() throws Exception { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerLaunchFromPreviousRM"); - super.testStartContainerFailureWithUnknownAuxService(); - } - - @Override - public void testIncreaseContainerResourceWithInvalidRequests() throws Exception { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testIncreaseContainerResourceWithInvalidRequests"); - super.testIncreaseContainerResourceWithInvalidRequests(); - } - - @Override - public void testIncreaseContainerResourceWithInvalidResource() throws Exception { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testIncreaseContainerResourceWithInvalidResource"); - super.testIncreaseContainerResourceWithInvalidResource(); - } - - @Override - public void testChangeContainerResource() throws Exception { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testChangeContainerResource"); - super.testChangeContainerResource(); - } - - @Override - public void testOutputThreadDumpSignal() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testOutputThreadDumpSignal"); - super.testOutputThreadDumpSignal(); - } - - @Override - public void testGracefulShutdownSignal() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testGracefulShutdownSignal"); - super.testGracefulShutdownSignal(); - } - - @Override - public void testForcefulShutdownSignal() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testForcefulShutdownSignal"); - super.testForcefulShutdownSignal(); - } - - @Override - public void testContainerUpgradeSuccessAutoCommit() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerUpgradeSuccessAutoCommit"); - super.testContainerUpgradeSuccessAutoCommit(); - } - - @Override - public void testContainerUpgradeLocalizationFailure() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerUpgradeLocalizationFailure"); - super.testContainerUpgradeLocalizationFailure(); - } - - @Override - public void testContainerUpgradeSuccessExplicitCommit() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerUpgradeSuccessExplicitCommit"); - super.testContainerUpgradeSuccessExplicitCommit(); - } - - @Override - public void testContainerUpgradeSuccessExplicitRollback() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerUpgradeSuccessExplicitRollback"); - super.testContainerUpgradeSuccessExplicitRollback(); - } - - @Override - public void testContainerUpgradeRollbackDueToFailure() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerUpgradeRollbackDueToFailure"); - super.testContainerUpgradeRollbackDueToFailure(); - } - - @Override - public void testContainerUpgradeProcessFailure() throws IOException, - InterruptedException, YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerUpgradeProcessFailure"); - super.testContainerUpgradeProcessFailure(); - } - - @Override - public void testContainerRestart() throws IOException, InterruptedException, - YarnException { - // Don't run the test if the binary is not available. - if (!shouldRunTest()) { - LOG.info("LCE binary path is not passed. Not running the test"); - return; - } - LOG.info("Running testContainerRestart"); - super.testContainerRestart(); - } - private boolean shouldRunTest() { return System .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java index 04cfae9f390..5ab5c3754f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java @@ -638,7 +638,7 @@ public int getNMRegistrationCount() { } @Override - protected void shutDown() { + protected void shutDown(int exitCode) { synchronized (isNMShutdownCalled) { isNMShutdownCalled.set(true); isNMShutdownCalled.notify(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 66686a53578..6fead7ef4c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -1859,4 +1859,49 @@ private void testContainerLaunchAndSignal(SignalContainerCommand command) Assert.assertEquals(signal, signalContext.getSignal()); } } + + @Test + public void testStartContainerFailureWithInvalidLocalResource() + throws Exception { + containerManager.start(); + LocalResource rsrc_alpha = + recordFactory.newRecordInstance(LocalResource.class); + rsrc_alpha.setResource(null); + rsrc_alpha.setSize(-1); + rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION); + rsrc_alpha.setType(LocalResourceType.FILE); + rsrc_alpha.setTimestamp(System.currentTimeMillis()); + Map localResources = + new HashMap(); + localResources.put("invalid_resource", rsrc_alpha); + ContainerLaunchContext containerLaunchContext = + recordFactory.newRecordInstance(ContainerLaunchContext.class); + ContainerLaunchContext spyContainerLaunchContext = + Mockito.spy(containerLaunchContext); + Mockito.when(spyContainerLaunchContext.getLocalResources()) + .thenReturn(localResources); + + ContainerId cId = createContainerId(0); + String user = "start_container_fail"; + Token containerToken = + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager()); + StartContainerRequest request = StartContainerRequest + .newInstance(spyContainerLaunchContext, containerToken); + + // start containers + List startRequest = + new ArrayList(); + startRequest.add(request); + StartContainersRequest requestList = + StartContainersRequest.newInstance(startRequest); + + StartContainersResponse response = + containerManager.startContainers(requestList); + Assert.assertTrue(response.getFailedRequests().size() == 1); + Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0); + Assert.assertTrue(response.getFailedRequests().containsKey(cId)); + Assert.assertTrue(response.getFailedRequests().get(cId).getMessage() + .contains("Null resource URL for local resource")); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java index ff19e9ba317..663113d80f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java @@ -99,6 +99,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler; +import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; @@ -699,8 +700,10 @@ private void waitForAppState(Application app, ApplicationState state) private ContainerManagerImpl createContainerManager(Context context) { final LogHandler logHandler = mock(LogHandler.class); + final NodeManagerMetrics metrics = mock(NodeManagerMetrics.class); final ResourceLocalizationService rsrcSrv = - new ResourceLocalizationService(null, null, null, null, context) { + new ResourceLocalizationService(null, null, null, null, context, + metrics) { @Override public void serviceInit(Configuration conf) throws Exception { } @@ -739,8 +742,10 @@ protected LogHandler createLogHandler(Configuration conf, } @Override - protected ResourceLocalizationService createResourceLocalizationService( - ContainerExecutor exec, DeletionService deletionContext, Context context) { + protected ResourceLocalizationService + createResourceLocalizationService( + ContainerExecutor exec, DeletionService deletionContext, + Context context, NodeManagerMetrics metrics) { return rsrcSrv; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheCleanup.java index d6db67a9db1..9f5b23c0d25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheCleanup.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheCleanup.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheCleaner.LocalCacheCleanerStats; +import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.junit.Test; /** @@ -80,8 +81,12 @@ public void testBasicCleanup() { ((StubbedLocalResourcesTrackerImpl) privateRsrc.get("user2")) .getLocalRsrc().size()); assertEquals(100, stats.getTotalDelSize()); + assertEquals(100, rls.metrics.getTotalBytesDeleted()); assertEquals(60, stats.getPublicDelSize()); + assertEquals(60, rls.metrics.getPublicBytesDeleted()); assertEquals(40, stats.getPrivateDelSize()); + assertEquals(40, rls.metrics.getPrivateBytesDeleted()); + assertEquals(100, rls.metrics.getCacheSizeBeforeClean()); } @Test @@ -105,8 +110,12 @@ public void testPositiveRefCount() { assertEquals(1, resources.getLocalRsrc().size()); assertTrue(resources.getLocalRsrc().containsKey(survivor)); assertEquals(20, stats.getTotalDelSize()); + assertEquals(20, rls.metrics.getTotalBytesDeleted()); assertEquals(20, stats.getPublicDelSize()); + assertEquals(20, rls.metrics.getPublicBytesDeleted()); assertEquals(0, stats.getPrivateDelSize()); + assertEquals(0, rls.metrics.getPrivateBytesDeleted()); + assertEquals(40, rls.metrics.getCacheSizeBeforeClean()); } @Test @@ -164,8 +173,12 @@ public void testLRUAcrossTrackers() { assertTrue(usr2LocalRsrc.containsKey(usr2Surviver1)); assertEquals(80, stats.getTotalDelSize()); + assertEquals(80, rls.metrics.getTotalBytesDeleted()); assertEquals(20, stats.getPublicDelSize()); + assertEquals(20, rls.metrics.getPublicBytesDeleted()); assertEquals(60, stats.getPrivateDelSize()); + assertEquals(60, rls.metrics.getPrivateBytesDeleted()); + assertEquals(160, rls.metrics.getCacheSizeBeforeClean()); } private ResourceLocalizationService createLocService( @@ -174,8 +187,10 @@ private ResourceLocalizationService createLocService( long targetCacheSize) { Context mockedContext = mock(Context.class); when(mockedContext.getNMStateStore()).thenReturn(null); + NodeManagerMetrics metrics = NodeManagerMetrics.create(); ResourceLocalizationService rls = - new ResourceLocalizationService(null, null, null, null, mockedContext); + new ResourceLocalizationService(null, null, null, null, mockedContext, + metrics); // We set the following members directly so we don't have to deal with // mocking out the service init method. rls.publicRsrc = new StubbedLocalResourcesTrackerImpl(null, publicRsrcs); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java index 82d9a946661..95cca2ca3b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java @@ -18,13 +18,15 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; -import org.junit.Assert; +import static org.mockito.Mockito.mock; +import org.junit.Assert; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheDirectoryManager.Directory; +import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; @@ -83,8 +85,10 @@ public void testMinimumPerDirectoryFileLimit() { new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), new NMNullStateStoreService(), false, conf); + NodeManagerMetrics metrics = mock(NodeManagerMetrics.class); ResourceLocalizationService service = - new ResourceLocalizationService(null, null, null, null, nmContext); + new ResourceLocalizationService(null, null, null, null, nmContext, + metrics); try { service.init(conf); } catch (Exception e1) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index 41cccc145fa..5b97e086c7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext; import org.junit.Assert; - import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; @@ -141,6 +140,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent; +import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; @@ -171,6 +171,7 @@ public class TestResourceLocalizationService { private AbstractFileSystem spylfs; private FileContext lfs; private NMContext nmContext; + private NodeManagerMetrics metrics; @BeforeClass public static void setupClass() { mockServer = mock(Server.class); @@ -189,6 +190,7 @@ public void setup() throws IOException { conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), new NMNullStateStoreService(), false, conf); + metrics = mock(NodeManagerMetrics.class); } @After @@ -225,7 +227,7 @@ public void testLocalizationInit() throws Exception { ResourceLocalizationService locService = spy(new ResourceLocalizationService(dispatcher, exec, delService, - diskhandler, nmContext)); + diskhandler, nmContext, metrics)); doReturn(lfs) .when(locService).getLocalFileContext(isA(Configuration.class)); try { @@ -286,7 +288,7 @@ public void testDirectoryCleanupOnNewlyCreatedStateStore() ResourceLocalizationService locService = spy(new ResourceLocalizationService(dispatcher, exec, delService, - diskhandler,nmContext)); + diskhandler, nmContext, metrics)); doReturn(lfs) .when(locService).getLocalFileContext(isA(Configuration.class)); try { @@ -357,7 +359,7 @@ public void testResourceRelease() throws Exception { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - dirsHandler, nmContext); + dirsHandler, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker( @@ -757,7 +759,7 @@ public void testLocalizerRunnerException() throws Exception { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - dirsHandlerSpy, nmContext); + dirsHandlerSpy, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); try { @@ -847,7 +849,7 @@ public void testLocalizationHeartbeat() throws Exception { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - dirsHandler, nmContext); + dirsHandler, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); @@ -1124,7 +1126,6 @@ private void setStopLocalization() { } @Test(timeout = 20000) - @SuppressWarnings("unchecked") public void testDownloadingResourcesOnContainerKill() throws Exception { List localDirs = new ArrayList(); String[] sDirs = new String[1]; @@ -1132,13 +1133,6 @@ public void testDownloadingResourcesOnContainerKill() throws Exception { sDirs[0] = localDirs.get(0).toString(); conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); - DrainDispatcher dispatcher = new DrainDispatcher(); - dispatcher.init(conf); - dispatcher.start(); - EventHandler applicationBus = mock(EventHandler.class); - dispatcher.register(ApplicationEventType.class, applicationBus); - EventHandler containerBus = mock(EventHandler.class); - dispatcher.register(ContainerEventType.class, containerBus); DummyExecutor exec = new DummyExecutor(); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); @@ -1149,8 +1143,10 @@ public void testDownloadingResourcesOnContainerKill() throws Exception { delService.init(new Configuration()); delService.start(); + DrainDispatcher dispatcher = getDispatcher(conf); ResourceLocalizationService rawService = new ResourceLocalizationService( - dispatcher, exec, delService, dirsHandler, nmContext); + dispatcher, exec, delService, dirsHandler, nmContext, metrics); + ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); @@ -1191,175 +1187,8 @@ public FileStatus answer(InvocationOnMock invocation) throws Throwable { spyService.init(conf); spyService.start(); - final Application app = mock(Application.class); - final ApplicationId appId = - BuilderUtils.newApplicationId(314159265358979L, 3); - String user = "user0"; - when(app.getUser()).thenReturn(user); - when(app.getAppId()).thenReturn(appId); - spyService.handle(new ApplicationLocalizationEvent( - LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); - ArgumentMatcher matchesAppInit = - new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - ApplicationEvent evt = (ApplicationEvent) o; - return evt.getType() == ApplicationEventType.APPLICATION_INITED - && appId == evt.getApplicationID(); - } - }; - dispatcher.await(); - verify(applicationBus).handle(argThat(matchesAppInit)); + doLocalization(spyService, dispatcher, exec, delService); - // Initialize localizer. - Random r = new Random(); - long seed = r.nextLong(); - System.out.println("SEED: " + seed); - r.setSeed(seed); - final Container c1 = getMockContainer(appId, 42, "user0"); - final Container c2 = getMockContainer(appId, 43, "user0"); - FSDataOutputStream out = - new FSDataOutputStream(new DataOutputBuffer(), null); - doReturn(out).when(spylfs).createInternal(isA(Path.class), - isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), - anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), - anyBoolean()); - final LocalResource resource1 = getPrivateMockedResource(r); - LocalResource resource2 = null; - do { - resource2 = getPrivateMockedResource(r); - } while (resource2 == null || resource2.equals(resource1)); - LocalResource resource3 = null; - do { - resource3 = getPrivateMockedResource(r); - } while (resource3 == null || resource3.equals(resource1) - || resource3.equals(resource2)); - - // Send localization requests for container c1 and c2. - final LocalResourceRequest req1 = new LocalResourceRequest(resource1); - final LocalResourceRequest req2 = new LocalResourceRequest(resource2); - final LocalResourceRequest req3 = new LocalResourceRequest(resource3); - Map> rsrcs = - new HashMap>(); - List privateResourceList = - new ArrayList(); - privateResourceList.add(req1); - privateResourceList.add(req2); - privateResourceList.add(req3); - rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); - spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs)); - - final LocalResourceRequest req1_1 = new LocalResourceRequest(resource2); - Map> rsrcs1 = - new HashMap>(); - List privateResourceList1 = - new ArrayList(); - privateResourceList1.add(req1_1); - rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1); - spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1)); - - dispatcher.await(); - // Wait for localizers of both container c1 and c2 to begin. - exec.waitForLocalizers(2); - LocalizerRunner locC1 = - spyService.getLocalizerRunner(c1.getContainerId().toString()); - final String containerIdStr = c1.getContainerId().toString(); - // Heartbeats from container localizer - LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class); - LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class); - LocalizerStatus stat = mock(LocalizerStatus.class); - when(stat.getLocalizerId()).thenReturn(containerIdStr); - when(rsrc1success.getResource()).thenReturn(resource1); - when(rsrc2pending.getResource()).thenReturn(resource2); - when(rsrc1success.getLocalSize()).thenReturn(4344L); - URL locPath = getPath("/some/path"); - when(rsrc1success.getLocalPath()).thenReturn(locPath); - when(rsrc1success.getStatus()). - thenReturn(ResourceStatusType.FETCH_SUCCESS); - when(rsrc2pending.getStatus()). - thenReturn(ResourceStatusType.FETCH_PENDING); - - when(stat.getResources()) - .thenReturn(Collections.emptyList()) - .thenReturn(Collections.singletonList(rsrc1success)) - .thenReturn(Collections.singletonList(rsrc2pending)) - .thenReturn(Collections.singletonList(rsrc2pending)) - .thenReturn(Collections.emptyList()); - - // First heartbeat which schedules first resource. - LocalizerHeartbeatResponse response = spyService.heartbeat(stat); - assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); - - // Second heartbeat which reports first resource as success. - // Second resource is scheduled. - response = spyService.heartbeat(stat); - assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); - final String locPath1 = response.getResourceSpecs().get(0). - getDestinationDirectory().getFile(); - - // Third heartbeat which reports second resource as pending. - // Third resource is scheduled. - response = spyService.heartbeat(stat); - assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); - final String locPath2 = response.getResourceSpecs().get(0). - getDestinationDirectory().getFile(); - - // Container c1 is killed which leads to cleanup - spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs)); - - // This heartbeat will indicate to container localizer to die as localizer - // runner has stopped. - response = spyService.heartbeat(stat); - assertEquals(LocalizerAction.DIE, response.getLocalizerAction()); - - exec.setStopLocalization(); - dispatcher.await(); - // verify container notification - ArgumentMatcher successContainerLoc = - new ArgumentMatcher() { - @Override - public boolean matches(Object o) { - ContainerEvent evt = (ContainerEvent) o; - return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED - && c1.getContainerId() == evt.getContainerID(); - } - }; - // Only one resource gets localized for container c1. - verify(containerBus).handle(argThat(successContainerLoc)); - - Set paths = - Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), - new Path(locPath2), new Path(locPath2 + "_tmp")); - // Wait for localizer runner thread for container c1 to finish. - while (locC1.getState() != Thread.State.TERMINATED) { - Thread.sleep(50); - } - // Verify if downloading resources were submitted for deletion. - verify(delService).delete(eq(user), - (Path) eq(null), argThat(new DownloadingPathsMatcher(paths))); - - LocalResourcesTracker tracker = spyService.getLocalResourcesTracker( - LocalResourceVisibility.PRIVATE, "user0", appId); - // Container c1 was killed but this resource was localized before kill - // hence its not removed despite ref cnt being 0. - LocalizedResource rsrc1 = tracker.getLocalizedResource(req1); - assertNotNull(rsrc1); - assertEquals(rsrc1.getState(), ResourceState.LOCALIZED); - assertEquals(rsrc1.getRefCount(), 0); - - // Container c1 was killed but this resource is referenced by container c2 - // as well hence its ref cnt is 1. - LocalizedResource rsrc2 = tracker.getLocalizedResource(req2); - assertNotNull(rsrc2); - assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING); - assertEquals(rsrc2.getRefCount(), 1); - - // As container c1 was killed and this resource was not referenced by any - // other container, hence its removed. - LocalizedResource rsrc3 = tracker.getLocalizedResource(req3); - assertNull(rsrc3); } finally { spyService.stop(); dispatcher.stop(); @@ -1367,6 +1196,228 @@ public boolean matches(Object o) { } } + private DrainDispatcher getDispatcher(Configuration config) { + DrainDispatcher dispatcher = new DrainDispatcher(); + dispatcher.init(config); + dispatcher.start(); + return dispatcher; + } + + @SuppressWarnings("unchecked") + private EventHandler getApplicationBus( + DrainDispatcher dispatcher) { + EventHandler applicationBus = mock(EventHandler.class); + dispatcher.register(ApplicationEventType.class, applicationBus); + return applicationBus; + } + + @SuppressWarnings("unchecked") + private EventHandler getContainerBus( + DrainDispatcher dispatcher) { + EventHandler containerBus = mock(EventHandler.class); + dispatcher.register(ContainerEventType.class, containerBus); + return containerBus; + } + + private void initApp(ResourceLocalizationService spyService, + EventHandler applicationBus, Application app, + ApplicationId appId, DrainDispatcher dispatcher) { + spyService.handle(new ApplicationLocalizationEvent( + LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); + ArgumentMatcher matchesAppInit = + new ArgumentMatcher() { + @Override + public boolean matches(Object o) { + ApplicationEvent evt = (ApplicationEvent) o; + return evt.getType() == ApplicationEventType.APPLICATION_INITED + && appId == evt.getApplicationID(); + } + }; + dispatcher.await(); + verify(applicationBus).handle(argThat(matchesAppInit)); + } + + private void doLocalization(ResourceLocalizationService spyService, + DrainDispatcher dispatcher, DummyExecutor exec, + DeletionService delService) + throws IOException, URISyntaxException, InterruptedException { + final Application app = mock(Application.class); + final ApplicationId appId = + BuilderUtils.newApplicationId(314159265358979L, 3); + String user = "user0"; + when(app.getUser()).thenReturn(user); + when(app.getAppId()).thenReturn(appId); + List resources = initializeLocalizer(appId); + LocalResource resource1 = resources.get(0); + LocalResource resource2 = resources.get(1); + LocalResource resource3 = resources.get(2); + final Container c1 = getMockContainer(appId, 42, "user0"); + final Container c2 = getMockContainer(appId, 43, "user0"); + + EventHandler applicationBus = + getApplicationBus(dispatcher); + EventHandler containerBus = getContainerBus(dispatcher); + initApp(spyService, applicationBus, app, appId, dispatcher); + + // Send localization requests for container c1 and c2. + final LocalResourceRequest req1 = new LocalResourceRequest(resource1); + final LocalResourceRequest req2 = new LocalResourceRequest(resource2); + final LocalResourceRequest req3 = new LocalResourceRequest(resource3); + Map> rsrcs = + new HashMap>(); + List privateResourceList = + new ArrayList(); + privateResourceList.add(req1); + privateResourceList.add(req2); + privateResourceList.add(req3); + rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); + spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs)); + + final LocalResourceRequest req11 = new LocalResourceRequest(resource2); + Map> rsrcs1 = + new HashMap>(); + List privateResourceList1 = + new ArrayList(); + privateResourceList1.add(req11); + rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1); + spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1)); + + dispatcher.await(); + // Wait for localizers of both container c1 and c2 to begin. + exec.waitForLocalizers(2); + LocalizerRunner locC1 = + spyService.getLocalizerRunner(c1.getContainerId().toString()); + + LocalizerStatus stat = mockLocalizerStatus(c1, resource1, resource2); + + // First heartbeat which schedules first resource. + LocalizerHeartbeatResponse response = spyService.heartbeat(stat); + assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); + + // Second heartbeat which reports first resource as success. + // Second resource is scheduled. + response = spyService.heartbeat(stat); + assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); + final String locPath1 = + response.getResourceSpecs().get(0).getDestinationDirectory().getFile(); + + // Third heartbeat which reports second resource as pending. + // Third resource is scheduled. + response = spyService.heartbeat(stat); + assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); + final String locPath2 = + response.getResourceSpecs().get(0).getDestinationDirectory().getFile(); + + // Container c1 is killed which leads to cleanup + spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs)); + + // This heartbeat will indicate to container localizer to die as localizer + // runner has stopped. + response = spyService.heartbeat(stat); + assertEquals(LocalizerAction.DIE, response.getLocalizerAction()); + + exec.setStopLocalization(); + dispatcher.await(); + // verify container notification + ArgumentMatcher successContainerLoc = + new ArgumentMatcher() { + @Override + public boolean matches(Object o) { + ContainerEvent evt = (ContainerEvent) o; + return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED + && c1.getContainerId() == evt.getContainerID(); + } + }; + // Only one resource gets localized for container c1. + verify(containerBus).handle(argThat(successContainerLoc)); + + Set paths = + Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), + new Path(locPath2), new Path(locPath2 + "_tmp")); + // Wait for localizer runner thread for container c1 to finish. + while (locC1.getState() != Thread.State.TERMINATED) { + Thread.sleep(50); + } + // Verify if downloading resources were submitted for deletion. + verify(delService).delete(eq(user), (Path) eq(null), + argThat(new DownloadingPathsMatcher(paths))); + + LocalResourcesTracker tracker = spyService.getLocalResourcesTracker( + LocalResourceVisibility.PRIVATE, "user0", appId); + // Container c1 was killed but this resource was localized before kill + // hence its not removed despite ref cnt being 0. + LocalizedResource rsrc1 = tracker.getLocalizedResource(req1); + assertNotNull(rsrc1); + assertEquals(rsrc1.getState(), ResourceState.LOCALIZED); + assertEquals(rsrc1.getRefCount(), 0); + + // Container c1 was killed but this resource is referenced by container c2 + // as well hence its ref cnt is 1. + LocalizedResource rsrc2 = tracker.getLocalizedResource(req2); + assertNotNull(rsrc2); + assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING); + assertEquals(rsrc2.getRefCount(), 1); + + // As container c1 was killed and this resource was not referenced by any + // other container, hence its removed. + LocalizedResource rsrc3 = tracker.getLocalizedResource(req3); + assertNull(rsrc3); + } + + private LocalizerStatus mockLocalizerStatus(Container c1, + LocalResource resource1, LocalResource resource2) { + final String containerIdStr = c1.getContainerId().toString(); + // Heartbeats from container localizer + LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class); + LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class); + LocalizerStatus stat = mock(LocalizerStatus.class); + when(stat.getLocalizerId()).thenReturn(containerIdStr); + when(rsrc1success.getResource()).thenReturn(resource1); + when(rsrc2pending.getResource()).thenReturn(resource2); + when(rsrc1success.getLocalSize()).thenReturn(4344L); + URL locPath = getPath("/some/path"); + when(rsrc1success.getLocalPath()).thenReturn(locPath); + when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); + when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING); + + when(stat.getResources()) + .thenReturn(Collections. emptyList()) + .thenReturn(Collections.singletonList(rsrc1success)) + .thenReturn(Collections.singletonList(rsrc2pending)) + .thenReturn(Collections.singletonList(rsrc2pending)) + .thenReturn(Collections. emptyList()); + return stat; + } + + @SuppressWarnings("unchecked") + private List initializeLocalizer(ApplicationId appId) + throws IOException { + // Initialize localizer. + Random r = new Random(); + long seed = r.nextLong(); + System.out.println("SEED: " + seed); + r.setSeed(seed); + FSDataOutputStream out = + new FSDataOutputStream(new DataOutputBuffer(), null); + doReturn(out).when(spylfs).createInternal(isA(Path.class), + isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), + anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), + anyBoolean()); + final LocalResource resource1 = getPrivateMockedResource(r); + LocalResource resource2 = null; + do { + resource2 = getPrivateMockedResource(r); + } while (resource2 == null || resource2.equals(resource1)); + LocalResource resource3 = null; + do { + resource3 = getPrivateMockedResource(r); + } while (resource3 == null || resource3.equals(resource1) + || resource3.equals(resource2)); + return Arrays.asList(resource1, resource2, resource3); + } + @Test @SuppressWarnings("unchecked") public void testPublicResourceInitializesLocalDir() throws Exception { @@ -1404,7 +1455,7 @@ public void testPublicResourceInitializesLocalDir() throws Exception { try { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - dirsHandler, spyContext); + dirsHandler, spyContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext( @@ -1499,7 +1550,8 @@ public void testLocalizerHeartbeatWhenAppCleaningUp() throws Exception { dirsHandler.init(conf); // Start resource localization service. ResourceLocalizationService rawService = new ResourceLocalizationService( - dispatcher, exec, mock(DeletionService.class), dirsHandler, nmContext); + dispatcher, exec, mock(DeletionService.class), dirsHandler, nmContext, + metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService). @@ -1618,7 +1670,7 @@ public void testFailedPublicResource() throws Exception { try { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - dirsHandler, nmContext); + dirsHandler, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext( @@ -1727,7 +1779,7 @@ public void testPublicResourceAddResourceExceptions() throws Exception { try { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - dirsHandlerSpy, nmContext); + dirsHandlerSpy, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext( @@ -1859,7 +1911,7 @@ public void testParallelDownloadAttemptsForPrivateResource() throws Exception { ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, - localDirHandler, nmContext); + localDirHandler, nmContext, metrics); dispatcher1.register(LocalizationEventType.class, rls); rls.init(conf); @@ -2012,7 +2064,7 @@ public void testLocalResourcePath() throws Exception { ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, - localDirHandler, nmContext); + localDirHandler, nmContext, metrics); dispatcher1.register(LocalizationEventType.class, rls); rls.init(conf); @@ -2178,7 +2230,7 @@ public void testParallelDownloadAttemptsForPublicResource() throws Exception { // it as otherwise it will remove requests from pending queue. ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher1, exec, delService, - dirsHandler, nmContext); + dirsHandler, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); dispatcher1.register(LocalizationEventType.class, spyService); spyService.init(conf); @@ -2484,7 +2536,7 @@ private ResourceLocalizationService createSpyService( new ApplicationACLsManager(conf), stateStore, false, conf); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - dirsHandler, nmContext); + dirsHandler, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker( @@ -2548,7 +2600,7 @@ public void testFailedDirsResourceRelease() throws Exception { // setup mocks ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, - mockDirsHandler, nmContext); + mockDirsHandler, nmContext, metrics); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java index 17d527a7a5d..097146b1e99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java @@ -416,8 +416,7 @@ public AppLogAggregatorInTest(Dispatcher dispatcher, logAggregationContext, context, lfs, -1, recoveredLogInitedTime); this.applicationId = appId; this.deletionService = deletionService; - - this.logWriter = getSpiedLogWriter(conf, ugi, remoteNodeLogFileForApp); + this.logWriter = spy(new LogWriter()); this.logValue = ArgumentCaptor.forClass(LogValue.class); } @@ -425,10 +424,5 @@ public AppLogAggregatorInTest(Dispatcher dispatcher, protected LogWriter createLogWriter() { return this.logWriter; } - - private LogWriter getSpiedLogWriter(Configuration conf, - UserGroupInformation ugi, Path remoteAppLogFile) throws IOException { - return spy(new LogWriter(conf, remoteAppLogFile, ugi)); - } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml index 47db01fbc1b..785ed0453b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml @@ -111,9 +111,27 @@ 40 Number of missed scheduling opportunities after which the CapacityScheduler - attempts to schedule rack-local containers. - Typically this should be set to number of nodes in the cluster, By default is setting - approximately number of nodes in one rack which is 40. + attempts to schedule rack-local containers. + When setting this parameter, the size of the cluster should be taken into account. + We use 40 as the default value, which is approximately the number of nodes in one rack. + + + + + yarn.scheduler.capacity.rack-locality-additional-delay + -1 + + Number of additional missed scheduling opportunities over the node-locality-delay + ones, after which the CapacityScheduler attempts to schedule off-switch containers, + instead of rack-local ones. + Example: with node-locality-delay=40 and rack-locality-delay=20, the scheduler will + attempt rack-local assignments after 40 missed opportunities, and off-switch assignments + after 40+20=60 missed opportunities. + When setting this parameter, the size of the cluster should be taken into account. + We use -1 as the default value, which disables this feature. In this case, the number + of missed opportunities for assigning off-switch containers is calculated based on + the number of containers and unique locations specified in the resource request, + as well as the size of the cluster. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 0a85d0c624b..533be9ef61b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -91,6 +91,11 @@ hadoop-hdfs test + + org.apache.hadoop + hadoop-hdfs-client + test + com.google.inject guice diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 412d573dbd2..a8bf6bd421d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -179,8 +179,9 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) { getTimelinelineAppMetrics(appMetrics, finishedTime); entity.setMetrics(entityMetrics); - getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( - SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); + getDispatcher().getEventHandler().handle( + new ApplicationFinishPublishEvent(SystemMetricsEventType. + PUBLISH_APPLICATION_FINISHED_ENTITY, entity, app)); } private Set getTimelinelineAppMetrics( @@ -452,16 +453,16 @@ private void putEntity(TimelineEntity entity, ApplicationId appId) { } private class ApplicationFinishPublishEvent extends TimelineV2PublishEvent { - private RMAppImpl app; + private RMApp app; public ApplicationFinishPublishEvent(SystemMetricsEventType type, - TimelineEntity entity, RMAppImpl app) { + TimelineEntity entity, RMApp app) { super(type, entity, app.getApplicationId()); this.app = app; } public RMAppImpl getRMAppImpl() { - return app; + return (RMAppImpl) app; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index effe422bf7d..79b25edc14e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; @@ -133,6 +134,17 @@ public void removeFromClusterNodeLabels(Collection labelsToRemove) } } + @Override + public void addToCluserNodeLabels(Collection labels) + throws IOException { + try { + writeLock.lock(); + super.addToCluserNodeLabels(labels); + } finally { + writeLock.unlock(); + } + } + @Override public void removeLabelsFromNode(Map> removeLabelsFromNode) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 929e2dab119..95919451a73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -785,7 +785,7 @@ private FileStatus getFileStatus(Path path) throws Exception { * atomic for underlying file system. */ protected void writeFile(Path outputPath, byte[] data, boolean - makeUnradableByAdmin) throws Exception { + makeUnreadableByAdmin) throws Exception { Path tempPath = new Path(outputPath.getParent(), outputPath.getName() + ".tmp"); FSDataOutputStream fsOut = null; @@ -793,7 +793,7 @@ protected void writeFile(Path outputPath, byte[] data, boolean // final status. try { fsOut = fs.create(tempPath, true); - if (makeUnradableByAdmin) { + if (makeUnreadableByAdmin) { setUnreadableBySuperuserXattrib(tempPath); } fsOut.write(data); @@ -811,10 +811,10 @@ protected void writeFile(Path outputPath, byte[] data, boolean * atomic for underlying file system. */ protected void updateFile(Path outputPath, byte[] data, boolean - makeUnradableByAdmin) throws Exception { + makeUnreadableByAdmin) throws Exception { Path newPath = new Path(outputPath.getParent(), outputPath.getName() + ".new"); // use writeFileWithRetries to make sure .new file is created atomically - writeFileWithRetries(newPath, data, makeUnradableByAdmin); + writeFileWithRetries(newPath, data, makeUnreadableByAdmin); replaceFile(newPath, outputPath); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java index 7ad381ed84f..29680e5d87b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java @@ -42,7 +42,8 @@ * when resources are being reserved to fill space for a future container * allocation. */ -public interface RMContainer extends EventHandler { +public interface RMContainer extends EventHandler, + Comparable { ContainerId getContainerId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 12fbbea72ac..1e9463a3e01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -63,7 +63,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @SuppressWarnings({"unchecked", "rawtypes"}) -public class RMContainerImpl implements RMContainer, Comparable { +public class RMContainerImpl implements RMContainer { private static final Log LOG = LogFactory.getLog(RMContainerImpl.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index bff9c4131f8..4de5eace54e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -25,12 +25,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; @@ -51,9 +47,8 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -86,8 +81,8 @@ public class AppSchedulingInfo { private Set requestedPartitions = new HashSet<>(); - private final ConcurrentSkipListMap - schedulerKeys = new ConcurrentSkipListMap<>(); + private final ConcurrentSkipListSet + schedulerKeys = new ConcurrentSkipListSet<>(); final Map> schedulerKeyToPlacementSets = new ConcurrentHashMap<>(); @@ -156,29 +151,6 @@ private void clearRequests() { LOG.info("Application " + applicationId + " requests cleared"); } - - private void incrementSchedulerKeyReference( - SchedulerRequestKey schedulerKey) { - Integer schedulerKeyCount = schedulerKeys.get(schedulerKey); - if (schedulerKeyCount == null) { - schedulerKeys.put(schedulerKey, 1); - } else { - schedulerKeys.put(schedulerKey, schedulerKeyCount + 1); - } - } - - public void decrementSchedulerKeyReference( - SchedulerRequestKey schedulerKey) { - Integer schedulerKeyCount = schedulerKeys.get(schedulerKey); - if (schedulerKeyCount != null) { - if (schedulerKeyCount > 1) { - schedulerKeys.put(schedulerKey, schedulerKeyCount - 1); - } else { - schedulerKeys.remove(schedulerKey); - } - } - } - public ContainerUpdateContext getUpdateContext() { return updateContext; } @@ -230,6 +202,10 @@ public boolean updateResourceRequests(List requests, } } + public void removePlacementSets(SchedulerRequestKey schedulerRequestKey) { + schedulerKeyToPlacementSets.remove(schedulerRequestKey); + } + boolean addToPlacementSets( boolean recoverPreemptedRequestForAContainer, Map> dedupRequests) { @@ -268,7 +244,8 @@ private void updatePendingResources(ResourceRequest lastRequest, (lastRequest != null) ? lastRequest.getNumContainers() : 0; if (request.getNumContainers() <= 0) { if (lastRequestContainers >= 0) { - decrementSchedulerKeyReference(schedulerKey); + schedulerKeys.remove(schedulerKey); + schedulerKeyToPlacementSets.remove(schedulerKey); } LOG.info("checking for deactivate of application :" + this.applicationId); @@ -276,7 +253,7 @@ private void updatePendingResources(ResourceRequest lastRequest, } else { // Activate application. Metrics activation is done here. if (lastRequestContainers <= 0) { - incrementSchedulerKeyReference(schedulerKey); + schedulerKeys.add(schedulerKey); abstractUsersManager.activateApplication(user, applicationId); } } @@ -366,7 +343,7 @@ public boolean getAndResetBlacklistChanged() { } public Collection getSchedulerKeys() { - return schedulerKeys.keySet(); + return schedulerKeys; } /** @@ -389,7 +366,7 @@ public List getAllResourceRequests() { public PendingAsk getNextPendingAsk() { try { readLock.lock(); - SchedulerRequestKey firstRequestKey = schedulerKeys.firstKey(); + SchedulerRequestKey firstRequestKey = schedulerKeys.first(); return getPendingAsk(firstRequestKey, ResourceRequest.ANY); } finally { readLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 91e29d59ab7..294897f5c30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -1304,6 +1304,11 @@ public SchedulingPlacementSet getSchedulingPlacemen return appSchedulingInfo.getSchedulingPlacementSet(schedulerRequestKey); } + public Map getResourceRequests( + SchedulerRequestKey schedulerRequestKey) { + return appSchedulingInfo.getSchedulingPlacementSet(schedulerRequestKey) + .getResourceRequests(); + } public void incUnconfirmedRes(Resource res) { unconfirmedAllocatedMem.addAndGet(res.getMemorySize()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index db17b42d021..272537c8bf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -159,7 +160,7 @@ public void allocateContainer(RMContainer rmContainer) { * @param rmContainer Allocated container * @param launchedOnNode True if the container has been launched */ - private synchronized void allocateContainer(RMContainer rmContainer, + protected synchronized void allocateContainer(RMContainer rmContainer, boolean launchedOnNode) { Container container = rmContainer.getContainer(); if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { @@ -286,7 +287,8 @@ private synchronized void addUnallocatedResource(Resource resource) { * container. * @param resource Resources to deduct. */ - private synchronized void deductUnallocatedResource(Resource resource) { + @VisibleForTesting + public synchronized void deductUnallocatedResource(Resource resource) { if (resource == null) { LOG.error("Invalid deduction of null resource for " + rmNode.getNodeAddress()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index aa60c9cbc16..16433909408 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -228,16 +228,6 @@ public boolean hasAccess(QueueACL acl, UserGroupInformation user) { null, null, Server.getRemoteAddress(), null)); } - @Override - public void setUsedCapacity(float usedCapacity) { - queueCapacities.setUsedCapacity(usedCapacity); - } - - @Override - public void setAbsoluteUsedCapacity(float absUsedCapacity) { - queueCapacities.setAbsoluteUsedCapacity(absUsedCapacity); - } - /** * Set maximum capacity - used only for testing. * @param maximumCapacity new max capacity @@ -309,7 +299,7 @@ void setupQueueConfigs(Resource clusterResource) // Update metrics CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, - minimumAllocation, this, labelManager, null); + this, labelManager, null); // Check if labels of this queue is a subset of parent queue, only do this // when we not root @@ -461,7 +451,7 @@ void allocateResource(Resource clusterResource, ++numContainers; CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, - minimumAllocation, this, labelManager, nodePartition); + this, labelManager, nodePartition); } finally { writeLock.unlock(); } @@ -474,7 +464,7 @@ protected void releaseResource(Resource clusterResource, queueUsage.decUsed(nodePartition, resource); CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, - minimumAllocation, this, labelManager, nodePartition); + this, labelManager, nodePartition); --numContainers; } finally { @@ -735,7 +725,7 @@ public void incUsedResource(String nodeLabel, Resource resourceToInc, queueUsage.incUsed(nodeLabel, resourceToInc); CSQueueUtils.updateUsedCapacity(resourceCalculator, labelManager.getResourceByLabel(nodeLabel, Resources.none()), - minimumAllocation, queueUsage, queueCapacities, nodeLabel); + nodeLabel, this); if (null != parent) { parent.incUsedResource(nodeLabel, resourceToInc, null); } @@ -751,7 +741,7 @@ public void decUsedResource(String nodeLabel, Resource resourceToDec, queueUsage.decUsed(nodeLabel, resourceToDec); CSQueueUtils.updateUsedCapacity(resourceCalculator, labelManager.getResourceByLabel(nodeLabel, Resources.none()), - minimumAllocation, queueUsage, queueCapacities, nodeLabel); + nodeLabel, this); if (null != parent) { parent.decUsedResource(nodeLabel, resourceToDec, null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 6d30386cd12..c6726ec6b62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -120,20 +120,6 @@ public interface CSQueue extends SchedulerQueue { */ public float getAbsoluteUsedCapacity(); - /** - * Set used capacity of the queue. - * @param usedCapacity - * used capacity of the queue - */ - public void setUsedCapacity(float usedCapacity); - - /** - * Set absolute used capacity of the queue. - * @param absUsedCapacity - * absolute used capacity of the queue - */ - public void setAbsoluteUsedCapacity(float absUsedCapacity); - /** * Get the current used capacity of nodes without label(s) of the queue * and it's children (if any). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java index 58c398410af..a601b7b805a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java @@ -23,6 +23,7 @@ import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableGaugeFloat; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; @@ -39,6 +40,10 @@ public class CSQueueMetrics extends QueueMetrics { MutableGaugeLong usedAMResourceMB; @Metric("Used AM CPU limit in virtual cores") MutableGaugeLong usedAMResourceVCores; + @Metric("Percent of Capacity Used") + MutableGaugeFloat usedCapacity; + @Metric("Percent of Absolute Capacity Used") + MutableGaugeFloat absoluteUsedCapacity; CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { @@ -91,6 +96,22 @@ public void decAMUsed(String user, Resource res) { } } + public float getUsedCapacity() { + return usedCapacity.value(); + } + + public void setUsedCapacity(float usedCapacity) { + this.usedCapacity.set(usedCapacity); + } + + public float getAbsoluteUsedCapacity() { + return absoluteUsedCapacity.value(); + } + + public void setAbsoluteUsedCapacity(Float absoluteUsedCapacity) { + this.absoluteUsedCapacity.set(absoluteUsedCapacity); + } + public synchronized static CSQueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index d5cdb3221f0..ba225413f6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import java.util.HashSet; import java.util.Set; import org.apache.hadoop.yarn.api.records.Resource; @@ -181,9 +180,12 @@ private static void updateAbsoluteCapacitiesByNodeLabels( * used resource for all partitions of this queue. */ public static void updateUsedCapacity(final ResourceCalculator rc, - final Resource totalPartitionResource, final Resource minimumAllocation, - ResourceUsage queueResourceUsage, QueueCapacities queueCapacities, - String nodePartition) { + final Resource totalPartitionResource, String nodePartition, + AbstractCSQueue childQueue) { + QueueCapacities queueCapacities = childQueue.getQueueCapacities(); + CSQueueMetrics queueMetrics = childQueue.getMetrics(); + ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage(); + Resource minimumAllocation = childQueue.getMinimumAllocation(); float absoluteUsedCapacity = 0.0f; float usedCapacity = 0.0f; float reservedCapacity = 0.0f; @@ -225,8 +227,18 @@ public static void updateUsedCapacity(final ResourceCalculator rc, queueCapacities.setReservedCapacity(nodePartition, reservedCapacity); queueCapacities .setAbsoluteReservedCapacity(nodePartition, absoluteReservedCapacity); + + // QueueMetrics does not support per-label capacities, + // so we report values only for the default partition. + if (nodePartition.equals(CommonNodeLabelsManager.NO_LABEL)) { + queueMetrics.setUsedCapacity( + queueCapacities.getUsedCapacity(RMNodeLabelsManager.NO_LABEL)); + queueMetrics.setAbsoluteUsedCapacity( + queueCapacities.getAbsoluteUsedCapacity( + RMNodeLabelsManager.NO_LABEL)); + } } - + private static Resource getMaxAvailableResourceToQueue( final ResourceCalculator rc, RMNodeLabelsManager nlm, CSQueue queue, Resource cluster) { @@ -270,22 +282,22 @@ private static Resource getMaxAvailableResourceToQueue( */ @Lock(CSQueue.class) public static void updateQueueStatistics( - final ResourceCalculator rc, final Resource cluster, final Resource minimumAllocation, - final CSQueue childQueue, final RMNodeLabelsManager nlm, + final ResourceCalculator rc, final Resource cluster, + final AbstractCSQueue childQueue, final RMNodeLabelsManager nlm, final String nodePartition) { QueueCapacities queueCapacities = childQueue.getQueueCapacities(); ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage(); - + if (nodePartition == null) { for (String partition : Sets.union( queueCapacities.getNodePartitionsSet(), queueResourceUsage.getNodePartitionsSet())) { updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster), - minimumAllocation, queueResourceUsage, queueCapacities, partition); + partition, childQueue); } } else { updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster), - minimumAllocation, queueResourceUsage, queueCapacities, nodePartition); + nodePartition, childQueue); } // Update queue metrics w.r.t node labels. In a generic way, we can diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 43ec3907333..9fb92ec1991 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -197,6 +197,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur @Private public static final int DEFAULT_NODE_LOCALITY_DELAY = 40; + @Private + public static final String RACK_LOCALITY_ADDITIONAL_DELAY = + PREFIX + "rack-locality-additional-delay"; + + @Private + public static final int DEFAULT_RACK_LOCALITY_ADDITIONAL_DELAY = -1; + @Private public static final String RACK_LOCALITY_FULL_RESET = PREFIX + "rack-locality-full-reset"; @@ -829,6 +836,11 @@ public int getNodeLocalityDelay() { return getInt(NODE_LOCALITY_DELAY, DEFAULT_NODE_LOCALITY_DELAY); } + public int getRackLocalityAdditionalDelay() { + return getInt(RACK_LOCALITY_ADDITIONAL_DELAY, + DEFAULT_RACK_LOCALITY_ADDITIONAL_DELAY); + } + public boolean getRackLocalityFullReset() { return getBoolean(RACK_LOCALITY_FULL_RESET, DEFAULT_RACK_LOCALITY_FULL_RESET); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 76cb5d6aac6..be6243d858e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -41,10 +41,13 @@ import org.apache.hadoop.yarn.security.Permission; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import com.google.common.annotations.VisibleForTesting; @@ -220,6 +223,23 @@ static CSQueue parseQueue( queue = new PlanQueue(csContext, queueName, parent, oldQueues.get(queueName)); + + //initializing the "internal" default queue, for SLS compatibility + String defReservationId = + queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX; + + List childQueues = new ArrayList<>(); + ReservationQueue resQueue = new ReservationQueue(csContext, + defReservationId, (PlanQueue) queue); + try { + resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f)); + } catch (SchedulerDynamicEditException e) { + throw new IllegalStateException(e); + } + childQueues.add(resQueue); + ((PlanQueue) queue).setChildQueues(childQueues); + queues.put(defReservationId, resQueue); + } else { queue = new LeafQueue(csContext, queueName, parent, @@ -264,6 +284,8 @@ static CSQueue parseQueue( /** * Ensure all existing queues are present. Queues cannot be deleted if its not * in Stopped state, Queue's cannot be moved from one hierarchy to other also. + * Previous child queue could be converted into parent queue if it is in + * STOPPED state. * * @param queues existing queues * @param newQueues new queues @@ -292,6 +314,17 @@ private void validateQueueHierarchy(Map queues, throw new IOException(queueName + " is moved from:" + oldQueue.getQueuePath() + " to:" + newQueue.getQueuePath() + " after refresh, which is not allowed."); + } else if (oldQueue instanceof LeafQueue + && newQueue instanceof ParentQueue) { + if (oldQueue.getState() == QueueState.STOPPED) { + LOG.info("Converting the leaf queue: " + oldQueue.getQueuePath() + + " to parent queue."); + } else { + throw new IOException("Can not convert the leaf queue: " + + oldQueue.getQueuePath() + " to parent queue since " + + "it is not yet in stopped state. Current State : " + + oldQueue.getState()); + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 1b20556ede7..9059ef0936e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -95,6 +95,7 @@ public class LeafQueue extends AbstractCSQueue { private float maxAMResourcePerQueuePercent; private volatile int nodeLocalityDelay; + private volatile int rackLocalityAdditionalDelay; private volatile boolean rackLocalityFullReset; Map applicationAttemptMap = @@ -215,6 +216,7 @@ protected void setupQueueConfigs(Resource clusterResource) } nodeLocalityDelay = conf.getNodeLocalityDelay(); + rackLocalityAdditionalDelay = conf.getRackLocalityAdditionalDelay(); rackLocalityFullReset = conf.getRackLocalityFullReset(); // re-init this since max allocation could have changed @@ -271,9 +273,12 @@ protected void setupQueueConfigs(Resource clusterResource) + "numContainers = " + numContainers + " [= currentNumContainers ]" + "\n" + "state = " + getState() + " [= configuredState ]" + "\n" + "acls = " + aclsString - + " [= configuredAcls ]" + "\n" + "nodeLocalityDelay = " - + nodeLocalityDelay + "\n" + "labels=" + labelStrBuilder - .toString() + "\n" + "reservationsContinueLooking = " + + " [= configuredAcls ]" + "\n" + + "nodeLocalityDelay = " + nodeLocalityDelay + "\n" + + "rackLocalityAdditionalDelay = " + + rackLocalityAdditionalDelay + "\n" + + "labels=" + labelStrBuilder.toString() + "\n" + + "reservationsContinueLooking = " + reservationsContinueLooking + "\n" + "preemptionDisabled = " + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = " + defaultAppPriorityPerQueue + "\npriority = " + priority); @@ -1346,6 +1351,11 @@ public int getNodeLocalityDelay() { return nodeLocalityDelay; } + @Lock(NoLock.class) + public int getRackLocalityAdditionalDelay() { + return rackLocalityAdditionalDelay; + } + @Lock(NoLock.class) public boolean getRackLocalityFullReset() { return rackLocalityFullReset; @@ -1666,7 +1676,7 @@ public void updateClusterResource(Resource clusterResource, // Update metrics CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, - minimumAllocation, this, labelManager, null); + this, labelManager, null); // queue metrics are updated, more resource may be available // activate the pending applications if possible diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index f84b7a42dbf..91fedbc14ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -315,7 +315,22 @@ public void reinitialize(CSQueue newlyParsedQueue, // Check if the child-queue already exists if (childQueue != null) { - // Re-init existing child queues + // Check if the child-queue has been converted into parent queue. + // The CS has already checked to ensure that this child-queue is in + // STOPPED state. + if (childQueue instanceof LeafQueue + && newChildQueue instanceof ParentQueue) { + // We would convert this LeafQueue to ParentQueue, consider this + // as the combination of DELETE then ADD. + newChildQueue.setParent(this); + currentChildQueues.put(newChildQueueName, newChildQueue); + // inform CapacitySchedulerQueueManager + CapacitySchedulerQueueManager queueManager = this.csContext + .getCapacitySchedulerQueueManager(); + queueManager.addQueue(newChildQueueName, newChildQueue); + continue; + } + // Re-init existing queues childQueue.reinitialize(newChildQueue, clusterResource); LOG.info(getQueueName() + ": re-configured queue: " + childQueue); } else{ @@ -825,7 +840,7 @@ public void updateClusterResource(Resource clusterResource, } CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, - minimumAllocation, this, labelManager, null); + this, labelManager, null); } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index a391f25fbae..882262fafcc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -93,10 +93,10 @@ public void reinitialize(CSQueue newlyParsedQueue, PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue; - if (newlyParsedParentQueue.getChildQueues().size() > 0) { + if (newlyParsedParentQueue.getChildQueues().size() != 1) { throw new IOException( "Reservable Queue should not have sub-queues in the" - + "configuration"); + + "configuration expect the default reservation queue"); } // Set new configs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java index faeb37e8f89..3d1b7317489 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java @@ -64,7 +64,7 @@ public void reinitialize(CSQueue newlyParsedQueue, } super.reinitialize(newlyParsedQueue, clusterResource); CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, - minimumAllocation, this, labelManager, null); + this, labelManager, null); updateQuotas(parent.getUserLimitForReservation(), parent.getUserLimitFactor(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index 8078bcd404d..f753d31fdbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -278,6 +278,12 @@ private int getActualNodeLocalityDelay() { .getCSLeafQueue().getNodeLocalityDelay()); } + private int getActualRackLocalityDelay() { + return Math.min(rmContext.getScheduler().getNumClusterNodes(), + application.getCSLeafQueue().getNodeLocalityDelay() + + application.getCSLeafQueue().getRackLocalityAdditionalDelay()); + } + private boolean canAssign(SchedulerRequestKey schedulerKey, FiCaSchedulerNode node, NodeType type, RMContainer reservedContainer) { @@ -286,26 +292,37 @@ private boolean canAssign(SchedulerRequestKey schedulerKey, if (reservedContainer != null) { return true; } + // If there are no nodes in the cluster, return false. + if (rmContext.getScheduler().getNumClusterNodes() == 0) { + return false; + } + // If we have only ANY requests for this schedulerKey, we should not + // delay its scheduling. + if (application.getResourceRequests(schedulerKey).size() == 1) { + return true; + } // 'Delay' off-switch long missedOpportunities = application.getSchedulingOpportunities(schedulerKey); - long requiredContainers = application.getOutstandingAsksCount( - schedulerKey); - float localityWaitFactor = - getLocalityWaitFactor(schedulerKey, rmContext.getScheduler() - .getNumClusterNodes()); - // Cap the delay by the number of nodes in the cluster. Under most - // conditions this means we will consider each node in the cluster before - // accepting an off-switch assignment. - return (Math.min(rmContext.getScheduler().getNumClusterNodes(), - (requiredContainers * localityWaitFactor)) < missedOpportunities); + // If rack locality additional delay parameter is enabled. + if (application.getCSLeafQueue().getRackLocalityAdditionalDelay() > -1) { + return missedOpportunities > getActualRackLocalityDelay(); + } else { + long requiredContainers = + application.getOutstandingAsksCount(schedulerKey); + float localityWaitFactor = getLocalityWaitFactor(schedulerKey, + rmContext.getScheduler().getNumClusterNodes()); + // Cap the delay by the number of nodes in the cluster. + return (Math.min(rmContext.getScheduler().getNumClusterNodes(), + (requiredContainers * localityWaitFactor)) < missedOpportunities); + } } // Check if we need containers on this rack - if (application.getOutstandingAsksCount(schedulerKey, node.getRackName()) - <= 0) { + if (application.getOutstandingAsksCount(schedulerKey, + node.getRackName()) <= 0) { return false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index ccfcffb83d7..a1c4b4b1027 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -74,11 +74,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt private static final DefaultResourceCalculator RESOURCE_CALCULATOR = new DefaultResourceCalculator(); - private long startTime; - private Priority appPriority; - private ResourceWeights resourceWeights; + private final long startTime; + private final Priority appPriority; + private final ResourceWeights resourceWeights; private Resource demand = Resources.createResource(0); - private FairScheduler scheduler; + private final FairScheduler scheduler; private FSQueue fsQueue; private Resource fairShare = Resources.createResource(0, 0); @@ -96,9 +96,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt // Used to record node reservation by an app. // Key = RackName, Value = Set of Nodes reserved by app on rack - private Map> reservations = new HashMap<>(); + private final Map> reservations = new HashMap<>(); - private List blacklistNodeIds = new ArrayList<>(); + private final List blacklistNodeIds = new ArrayList<>(); /** * Delay scheduling: We often want to prioritize scheduling of node-local * containers over rack-local or off-switch containers. To achieve this @@ -647,7 +647,11 @@ private boolean reserve(Resource perAllocationResource, FSSchedulerNode node, Container reservedContainer, NodeType type, SchedulerRequestKey schedulerKey) { - if (!reservationExceedsThreshold(node, type)) { + RMContainer nodeReservedContainer = node.getReservedContainer(); + boolean reservableForThisApp = nodeReservedContainer == null || + nodeReservedContainer.getApplicationAttemptId() + .equals(getApplicationAttemptId()); + if (reservableForThisApp &&!reservationExceedsThreshold(node, type)) { LOG.info("Making reservation: node=" + node.getNodeName() + " app_id=" + getApplicationId()); if (reservedContainer == null) { @@ -1139,7 +1143,7 @@ boolean isStarvedForFairShare() { /** * Is application starved for fairshare or minshare */ - private boolean isStarved() { + boolean isStarved() { return isStarvedForFairShare() || !Resources.isNone(minshareStarvation); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java index f6d843ae0bf..c841ca76ac9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java @@ -50,9 +50,6 @@ public class FSOpDurations implements MetricsSource { @Metric("Duration for a update thread run") MutableRate updateThreadRun; - @Metric("Duration for a preempt call") - MutableRate preemptCall; - private static final MetricsInfo RECORD_INFO = info("FSOpDurations", "Durations of FairScheduler calls or thread-runs"); @@ -84,7 +81,6 @@ private synchronized void setExtended(boolean isExtended) { continuousSchedulingRun.setExtended(isExtended); nodeUpdateCall.setExtended(isExtended); updateThreadRun.setExtended(isExtended); - preemptCall.setExtended(isExtended); INSTANCE.isExtended = isExtended; } @@ -106,10 +102,6 @@ public void addUpdateThreadRunDuration(long value) { updateThreadRun.add(value); } - public void addPreemptCallDuration(long value) { - preemptCall.add(value); - } - @VisibleForTesting public boolean hasUpdateThreadRunChanged() { return updateThreadRun.changed(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java index 65df0c2ea82..efe36a66af5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java @@ -113,11 +113,6 @@ private List identifyContainersToPreempt( List potentialNodes = scheduler.getNodeTracker() .getNodesByResourceName(rr.getResourceName()); for (FSSchedulerNode node : potentialNodes) { - // TODO (YARN-5829): Attempt to reserve the node for starved app. - if (isNodeAlreadyReserved(node, starvedApp)) { - continue; - } - int maxAMContainers = bestContainers == null ? Integer.MAX_VALUE : bestContainers.numAMContainers; PreemptableContainers preemptableContainers = @@ -134,7 +129,8 @@ private List identifyContainersToPreempt( if (bestContainers != null && bestContainers.containers.size() > 0) { containersToPreempt.addAll(bestContainers.containers); - trackPreemptionsAgainstNode(bestContainers.containers); + // Reserve the containers for the starved app + trackPreemptionsAgainstNode(bestContainers.containers, starvedApp); } } } // End of iteration over RRs @@ -163,8 +159,10 @@ private PreemptableContainers identifyContainersToPreemptOnNode( node.getRunningContainersWithAMsAtTheEnd(); containersToCheck.removeAll(node.getContainersForPreemption()); - // Initialize potential with unallocated resources - Resource potential = Resources.clone(node.getUnallocatedResource()); + // Initialize potential with unallocated but not reserved resources + Resource potential = Resources.subtractFromNonNegative( + Resources.clone(node.getUnallocatedResource()), + node.getTotalReserved()); for (RMContainer container : containersToCheck) { FSAppAttempt app = @@ -182,8 +180,6 @@ private PreemptableContainers identifyContainersToPreemptOnNode( // Check if we have already identified enough containers if (Resources.fitsIn(request, potential)) { return preemptableContainers; - } else { - // TODO (YARN-5829): Unreserve the node for the starved app. } } return null; @@ -195,10 +191,11 @@ private boolean isNodeAlreadyReserved( return nodeReservedApp != null && !nodeReservedApp.equals(app); } - private void trackPreemptionsAgainstNode(List containers) { + private void trackPreemptionsAgainstNode(List containers, + FSAppAttempt app) { FSSchedulerNode node = (FSSchedulerNode) scheduler.getNodeTracker() .getNode(containers.get(0).getNodeId()); - node.addContainersForPreemption(containers); + node.addContainersForPreemption(containers, app); } private void preemptContainers(List containers) { @@ -232,10 +229,6 @@ public void run() { LOG.info("Killing container " + container); scheduler.completedContainer( container, status, RMContainerEventType.KILL); - - FSSchedulerNode containerNode = (FSSchedulerNode) - scheduler.getNodeTracker().getNode(container.getAllocatedNode()); - containerNode.removeContainerForPreemption(container); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java index d983ea08237..663e3c8b3a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java @@ -18,18 +18,26 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.util.resource.Resources; import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; @@ -38,15 +46,38 @@ public class FSSchedulerNode extends SchedulerNode { private static final Log LOG = LogFactory.getLog(FSSchedulerNode.class); - private FSAppAttempt reservedAppSchedulable; - private final Set containersForPreemption = + // Stores list of containers still to be preempted + @VisibleForTesting + final Set containersForPreemption = new ConcurrentSkipListSet<>(); + // Stores amount of resources preempted and reserved for each app + @VisibleForTesting + final Map + resourcesPreemptedForApp = new LinkedHashMap<>(); + private final Map appIdToAppMap = + new HashMap<>(); + // Sum of resourcesPreemptedForApp values, total resources that are + // slated for preemption + private Resource totalResourcesPreempted = Resource.newInstance(0, 0); public FSSchedulerNode(RMNode node, boolean usePortForNodeName) { super(node, usePortForNodeName); } + /** + * Total amount of reserved resources including reservations and preempted + * containers. + * @return total resources reserved + */ + Resource getTotalReserved() { + Resource totalReserved = Resources.clone(getReservedContainer() != null + ? getReservedContainer().getAllocatedResource() + : Resource.newInstance(0, 0)); + Resources.addTo(totalReserved, totalResourcesPreempted); + return totalReserved; + } + @Override public synchronized void reserveResource( SchedulerApplicationAttempt application, SchedulerRequestKey schedulerKey, @@ -109,17 +140,56 @@ synchronized FSAppAttempt getReservedAppSchedulable() { return reservedAppSchedulable; } + /** + * List reserved resources after preemption and assign them to the + * appropriate applications in a FIFO order. + * @return if any resources were allocated + */ + @VisibleForTesting + synchronized LinkedHashMap getPreemptionList() { + cleanupPreemptionList(); + return new LinkedHashMap<>(resourcesPreemptedForApp); + } + + /** + * Remove apps that have their preemption requests fulfilled. + */ + private synchronized void cleanupPreemptionList() { + Iterator iterator = + resourcesPreemptedForApp.keySet().iterator(); + while (iterator.hasNext()) { + FSAppAttempt app = iterator.next(); + if (app.isStopped() || !app.isStarved()) { + // App does not need more resources + Resources.subtractFrom(totalResourcesPreempted, + resourcesPreemptedForApp.get(app)); + appIdToAppMap.remove(app.getApplicationAttemptId()); + iterator.remove(); + } + } + } + /** * Mark {@code containers} as being considered for preemption so they are * not considered again. A call to this requires a corresponding call to - * {@link #removeContainerForPreemption} to ensure we do not mark a - * container for preemption and never consider it again and avoid memory - * leaks. + * {@code releaseContainer} to ensure we do not mark a container for + * preemption and never consider it again and avoid memory leaks. * * @param containers container to mark */ - void addContainersForPreemption(Collection containers) { - containersForPreemption.addAll(containers); + void addContainersForPreemption(Collection containers, + FSAppAttempt app) { + + appIdToAppMap.putIfAbsent(app.getApplicationAttemptId(), app); + resourcesPreemptedForApp.putIfAbsent(app, Resource.newInstance(0, 0)); + Resource appReserved = resourcesPreemptedForApp.get(app); + + for(RMContainer container : containers) { + containersForPreemption.add(container); + Resources.addTo(appReserved, container.getAllocatedResource()); + Resources.addTo(totalResourcesPreempted, + container.getAllocatedResource()); + } } /** @@ -130,11 +200,50 @@ Set getContainersForPreemption() { } /** - * Remove container from the set of containers marked for preemption. - * - * @param container container to remove + * The Scheduler has allocated containers on this node to the given + * application. + * @param rmContainer Allocated container + * @param launchedOnNode True if the container has been launched */ - void removeContainerForPreemption(RMContainer container) { - containersForPreemption.remove(container); + @Override + protected synchronized void allocateContainer(RMContainer rmContainer, + boolean launchedOnNode) { + super.allocateContainer(rmContainer, launchedOnNode); + Resource allocated = rmContainer.getAllocatedResource(); + if (!Resources.isNone(allocated)) { + // check for satisfied preemption request and update bookkeeping + FSAppAttempt app = + appIdToAppMap.get(rmContainer.getApplicationAttemptId()); + if (app != null) { + Resource reserved = resourcesPreemptedForApp.get(app); + Resource fulfilled = Resources.componentwiseMin(reserved, allocated); + Resources.subtractFrom(reserved, fulfilled); + Resources.subtractFrom(totalResourcesPreempted, fulfilled); + if (Resources.isNone(reserved)) { + // No more preempted containers + resourcesPreemptedForApp.remove(app); + appIdToAppMap.remove(rmContainer.getApplicationAttemptId()); + } + } + } else { + LOG.error("Allocated empty container" + rmContainer.getContainerId()); + } + } + + /** + * Release an allocated container on this node. + * It also releases from the reservation list to trigger preemption + * allocations. + * @param containerId ID of container to be released. + * @param releasedByNode whether the release originates from a node update. + */ + @Override + public synchronized void releaseContainer(ContainerId containerId, + boolean releasedByNode) { + RMContainer container = getContainer(containerId); + super.releaseContainer(containerId, releasedByNode); + if (container != null) { + containersForPreemption.remove(container); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f3fde76f06a..d1a237ada97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -71,9 +71,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -913,8 +911,12 @@ protected void nodeUpdate(RMNode nm) { void continuousSchedulingAttempt() throws InterruptedException { long start = getClock().getTime(); - List nodeIdList = - nodeTracker.sortedNodeList(nodeAvailableResourceComparator); + List nodeIdList; + // Hold a lock to prevent comparator order changes due to changes of node + // unallocated resources + synchronized (this) { + nodeIdList = nodeTracker.sortedNodeList(nodeAvailableResourceComparator); + } // iterate all nodes for (FSSchedulerNode node : nodeIdList) { @@ -968,6 +970,31 @@ private boolean shouldContinueAssigning(int containers, } } + /** + * Assign preempted containers to the applications that have reserved + * resources for preempted containers. + * @param node Node to check + * @return assignment has occurred + */ + static boolean assignPreemptedContainers(FSSchedulerNode node) { + boolean assignedAny = false; + for (Entry entry : + node.getPreemptionList().entrySet()) { + FSAppAttempt app = entry.getKey(); + Resource preemptionPending = Resources.clone(entry.getValue()); + while (!app.isStopped() && !Resources.isNone(preemptionPending)) { + Resource assigned = app.assignContainer(node); + if (Resources.isNone(assigned)) { + // Fail to assign, let's not try further + break; + } + assignedAny = true; + Resources.subtractFromNonNegative(preemptionPending, assigned); + } + } + return assignedAny; + } + @VisibleForTesting void attemptScheduling(FSSchedulerNode node) { try { @@ -987,11 +1014,17 @@ void attemptScheduling(FSSchedulerNode node) { } // Assign new containers... - // 1. Check for reserved applications - // 2. Schedule if there are no reservations + // 1. Ensure containers are assigned to the apps that preempted + // 2. Check for reserved applications + // 3. Schedule if there are no reservations - boolean validReservation = false; + // Apps may wait for preempted containers + // We have to satisfy these first to avoid cases, when we preempt + // a container for A from B and C gets the preempted containers, + // when C does not qualify for preemption itself. + assignPreemptedContainers(node); FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable(); + boolean validReservation = false; if (reservedAppSchedulable != null) { validReservation = reservedAppSchedulable.assignReservedContainer(node); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java index c32246dd283..6cc8cc72031 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java @@ -204,15 +204,17 @@ public int getOutstandingAsksCount(String resourceName) { private void decrementOutstanding(SchedulerRequestKey schedulerRequestKey, ResourceRequest offSwitchRequest) { int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1; - - // Do not remove ANY offSwitchRequest.setNumContainers(numOffSwitchContainers); // Do we have any outstanding requests? // If there is nothing, we need to deactivate this application if (numOffSwitchContainers == 0) { - appSchedulingInfo.decrementSchedulerKeyReference(schedulerRequestKey); + appSchedulingInfo.getSchedulerKeys().remove(schedulerRequestKey); appSchedulingInfo.checkForDeactivation(); + resourceRequestMap.remove(ResourceRequest.ANY); + if (resourceRequestMap.isEmpty()) { + appSchedulingInfo.removePlacementSets(schedulerRequestKey); + } } appSchedulingInfo.decPendingResource( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java index 692924c01d6..2688987ed50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java @@ -69,18 +69,17 @@ public void testSignalRequestDeliveryToNM() throws Exception { //kick the scheduler nm1.nodeHeartbeat(true); - List conts = null; - int contReceived = 0; + List conts = new ArrayList<>(request); int waitCount = 0; - while (contReceived < request && waitCount++ < 200) { - LOG.info("Got " + contReceived + " containers. Waiting to get " + while (conts.size() < request && waitCount++ < 200) { + LOG.info("Got " + conts.size() + " containers. Waiting to get " + request); Thread.sleep(100); - conts = am.allocate(new ArrayList(), + List allocation = am.allocate(new ArrayList(), new ArrayList()).getAllocatedContainers(); - contReceived += conts.size(); + conts.addAll(allocation); } - Assert.assertEquals(request, contReceived); + Assert.assertEquals(request, conts.size()); for(Container container : conts) { rm.signalToContainer(container.getId(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java index bccd5c34a63..4320d3dc680 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java @@ -153,6 +153,13 @@ protected void verifyCapacity(Queue defQ) { assertTrue(csQueue.getCapacity() > 0.9); } + @Override + protected void checkDefaultQueueBeforePlanFollowerRun(){ + Queue defQ = getDefaultQueue(); + Assert.assertEquals(0, getNumberOfApplications(defQ)); + Assert.assertNotNull(defQ); + } + @Override protected Queue getDefaultQueue() { return cs.getQueue("dedicated" + ReservationConstants.DEFAULT_QUEUE_SUFFIX); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java index 1b4e2f8e135..9561234d633 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java @@ -131,6 +131,10 @@ public void testWithKillOnExpiry() throws PlanningException, testPlanFollower(false); } + @Override + protected void checkDefaultQueueBeforePlanFollowerRun() { + Assert.assertNull(getDefaultQueue()); + } @Override protected void verifyCapacity(Queue defQ) { assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > 0.9); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java index b604799c042..c6cebd96928 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java @@ -89,6 +89,11 @@ protected void testPlanFollower(boolean isMove) throws PlanningException, "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil .generateAllocation(10L, 1L, f2), res, minAlloc), false)); + + // default reseration queue should exist before run of PlanFollower AND have + // no apps + checkDefaultQueueBeforePlanFollowerRun(); + AbstractSchedulerPlanFollower planFollower = createPlanFollower(); when(mClock.getTime()).thenReturn(0L); @@ -108,8 +113,8 @@ protected void testPlanFollower(boolean isMove) throws PlanningException, new AppAttemptAddedSchedulerEvent(appAttemptId_0, false); scheduler.handle(appAttemptAddedEvent); - // initial default reservation queue should have no apps + // initial default reservation queue should have no apps after first run Queue defQ = getDefaultQueue(); Assert.assertEquals(0, getNumberOfApplications(defQ)); @@ -179,6 +184,8 @@ protected void testPlanFollower(boolean isMove) throws PlanningException, verifyCapacity(defQ); } + protected abstract void checkDefaultQueueBeforePlanFollowerRun(); + protected abstract Queue getReservationQueue(String reservationId); protected abstract void verifyCapacity(Queue defQ); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index ff0f7cfc377..447ee3d18f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -31,6 +31,7 @@ import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -63,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; @@ -86,6 +88,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.Application; +import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; @@ -472,6 +475,52 @@ private CapacitySchedulerConfiguration setupQueueConfigurationWithOutB1( return conf; } + /** + * @param conf, to be modified + * @return, CS configuration which has converted b1 to parent queue + * root + * / \ + * a b + * / \ / | \ + * a1 a2 b1 b2 b3 + * | + * b11 + */ + private CapacitySchedulerConfiguration + setupQueueConfigurationWithB1AsParentQueue( + CapacitySchedulerConfiguration conf) { + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] { "a", "b" }); + + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); + + // Define 2nd-level queues + conf.setQueues(A, new String[] { "a1", "a2" }); + conf.setCapacity(A1, A1_CAPACITY); + conf.setUserLimitFactor(A1, 100.0f); + conf.setCapacity(A2, A2_CAPACITY); + conf.setUserLimitFactor(A2, 100.0f); + + conf.setQueues(B, new String[] {"b1","b2", "b3"}); + conf.setCapacity(B1, B1_CAPACITY); + conf.setUserLimitFactor(B1, 100.0f); + conf.setCapacity(B2, B2_CAPACITY); + conf.setUserLimitFactor(B2, 100.0f); + conf.setCapacity(B3, B3_CAPACITY); + conf.setUserLimitFactor(B3, 100.0f); + + // Set childQueue for B1 + conf.setQueues(B1, new String[] {"b11"}); + String B11 = B1 + ".b11"; + conf.setCapacity(B11, 100.0f); + conf.setUserLimitFactor(B11, 100.0f); + + return conf; + } + /** * @param conf, to be modified * @return, CS configuration which has deleted a @@ -2900,6 +2949,162 @@ private void waitContainerAllocated(MockAM am, int mem, int nContainer, } } + @Test + public void testSchedulerKeyGarbageCollection() throws Exception { + YarnConfiguration conf = + new YarnConfiguration(new CapacitySchedulerConfiguration()); + conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true); + + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + MockRM rm = new MockRM(conf, memStore); + rm.start(); + + HashMap nodes = new HashMap<>(); + MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService()); + nodes.put(nm1.getNodeId(), nm1); + MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService()); + nodes.put(nm2.getNodeId(), nm2); + MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService()); + nodes.put(nm3.getNodeId(), nm3); + MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService()); + nodes.put(nm4.getNodeId(), nm4); + nm1.registerNode(); + nm2.registerNode(); + nm3.registerNode(); + nm4.registerNode(); + + RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); + ApplicationAttemptId attemptId = + app1.getCurrentAppAttempt().getAppAttemptId(); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2); + ResourceScheduler scheduler = rm.getResourceScheduler(); + + // All nodes 1 - 4 will be applicable for scheduling. + nm1.nodeHeartbeat(true); + nm2.nodeHeartbeat(true); + nm3.nodeHeartbeat(true); + nm4.nodeHeartbeat(true); + + Thread.sleep(1000); + + AllocateResponse allocateResponse = am1.allocate( + Arrays.asList( + newResourceRequest(1, 1, ResourceRequest.ANY, + Resources.createResource(3 * GB), 1, true, + ExecutionType.GUARANTEED), + newResourceRequest(2, 2, ResourceRequest.ANY, + Resources.createResource(3 * GB), 1, true, + ExecutionType.GUARANTEED), + newResourceRequest(3, 3, ResourceRequest.ANY, + Resources.createResource(3 * GB), 1, true, + ExecutionType.GUARANTEED), + newResourceRequest(4, 4, ResourceRequest.ANY, + Resources.createResource(3 * GB), 1, true, + ExecutionType.GUARANTEED) + ), + null); + List allocatedContainers = allocateResponse + .getAllocatedContainers(); + Assert.assertEquals(0, allocatedContainers.size()); + + Collection schedulerKeys = + ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId) + .getAppSchedulingInfo().getSchedulerKeys(); + Assert.assertEquals(4, schedulerKeys.size()); + + // Get a Node to HB... at which point 1 container should be + // allocated + nm1.nodeHeartbeat(true); + Thread.sleep(200); + allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>()); + allocatedContainers = allocateResponse.getAllocatedContainers(); + Assert.assertEquals(1, allocatedContainers.size()); + + // Verify 1 outstanding schedulerKey is removed + Assert.assertEquals(3, schedulerKeys.size()); + + List resReqs = + ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId) + .getAppSchedulingInfo().getAllResourceRequests(); + + // Verify 1 outstanding schedulerKey is removed from the + // rrMap as well + Assert.assertEquals(3, resReqs.size()); + + // Verify One more container Allocation on node nm2 + // And ensure the outstanding schedulerKeys go down.. + nm2.nodeHeartbeat(true); + Thread.sleep(200); + + // Update the allocateReq to send 0 numContainer req. + // For the satisfied container... + allocateResponse = am1.allocate(Arrays.asList( + newResourceRequest(1, + allocatedContainers.get(0).getAllocationRequestId(), + ResourceRequest.ANY, + Resources.createResource(3 * GB), 0, true, + ExecutionType.GUARANTEED) + ), + new ArrayList<>()); + allocatedContainers = allocateResponse.getAllocatedContainers(); + Assert.assertEquals(1, allocatedContainers.size()); + + // Verify 1 outstanding schedulerKey is removed + Assert.assertEquals(2, schedulerKeys.size()); + + resReqs = ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId) + .getAppSchedulingInfo().getAllResourceRequests(); + // Verify the map size is not increased due to 0 req + Assert.assertEquals(2, resReqs.size()); + + // Now Verify that the AM can cancel 1 Ask: + SchedulerRequestKey sk = schedulerKeys.iterator().next(); + am1.allocate( + Arrays.asList( + newResourceRequest(sk.getPriority().getPriority(), + sk.getAllocationRequestId(), + ResourceRequest.ANY, Resources.createResource(3 * GB), 0, true, + ExecutionType.GUARANTEED) + ), + null); + + schedulerKeys = + ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId) + .getAppSchedulingInfo().getSchedulerKeys(); + + Thread.sleep(200); + + // Verify 1 outstanding schedulerKey is removed because of the + // cancel ask + Assert.assertEquals(1, schedulerKeys.size()); + + // Now verify that after the next node heartbeat, we allocate + // the last schedulerKey + nm3.nodeHeartbeat(true); + Thread.sleep(200); + allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>()); + allocatedContainers = allocateResponse.getAllocatedContainers(); + Assert.assertEquals(1, allocatedContainers.size()); + + // Verify no more outstanding schedulerKeys.. + Assert.assertEquals(0, schedulerKeys.size()); + resReqs = + ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId) + .getAppSchedulingInfo().getAllResourceRequests(); + Assert.assertEquals(0, resReqs.size()); + } + + private static ResourceRequest newResourceRequest(int priority, + long allocReqId, String rName, Resource resource, int numContainers, + boolean relaxLoc, ExecutionType eType) { + ResourceRequest rr = ResourceRequest.newInstance( + Priority.newInstance(priority), rName, resource, numContainers, + relaxLoc, null, ExecutionTypeRequest.newInstance(eType, true)); + rr.setAllocationRequestId(allocReqId); + return rr; + } + @Test public void testHierarchyQueuesCurrentLimits() throws Exception { /* @@ -4142,4 +4347,61 @@ null, new RMContainerTokenSecretManager(conf), cs.stop(); } + + /** + * Test if we can convert a leaf queue to a parent queue + * @throws Exception + */ + @Test (timeout = 10000) + public void testConvertLeafQueueToParentQueue() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); + setupQueueConfiguration(conf); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + cs.reinitialize(conf, rmContext); + checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); + + String targetQueue = "b1"; + CSQueue b1 = cs.getQueue(targetQueue); + Assert.assertEquals(b1.getState(), QueueState.RUNNING); + + // test if we can convert a leaf queue which is in RUNNING state + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithB1AsParentQueue(conf); + try { + cs.reinitialize(conf, mockContext); + fail("Expected to throw exception when refresh queue tries to convert" + + " a child queue to a parent queue."); + } catch (IOException e) { + // ignore + } + + // now set queue state for b1 to STOPPED + conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.set("yarn.scheduler.capacity.root.b.b1.state", "STOPPED"); + cs.reinitialize(conf, mockContext); + Assert.assertEquals(b1.getState(), QueueState.STOPPED); + + // test if we can convert a leaf queue which is in STOPPED state + conf = new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithB1AsParentQueue(conf); + try { + cs.reinitialize(conf, mockContext); + } catch (IOException e) { + fail("Expected to NOT throw exception when refresh queue tries" + + " to convert a leaf queue WITHOUT running apps"); + } + b1 = cs.getQueue(targetQueue); + Assert.assertTrue(b1 instanceof ParentQueue); + Assert.assertEquals(b1.getState(), QueueState.RUNNING); + Assert.assertTrue(!b1.getChildQueues().isEmpty()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index ce3382fdb23..483ba1bc95d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -76,6 +76,12 @@ public void setUp() { public void testRefreshQueuesWithReservations() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + //set default queue capacity to zero + ((ReservationQueue) cs + .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX)) + .setEntitlement( + new QueueEntitlement(0f, 1f)); + // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); @@ -120,6 +126,11 @@ public void testAddQueueFailCases() throws Exception { ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a")); cs.addQueue(a1); + //set default queue capacity to zero + ((ReservationQueue) cs + .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX)) + .setEntitlement( + new QueueEntitlement(0f, 1f)); a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 1162b9fc7b4..a9ed5a90e67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -104,7 +104,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSchedulerKey; -public class TestLeafQueue { +public class TestLeafQueue { private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private static final Log LOG = LogFactory.getLog(TestLeafQueue.class); @@ -1059,13 +1059,9 @@ public void testComputeUserLimitAndSetHeadroom() throws IOException { //test case 3 qb.finishApplication(app_0.getApplicationId(), user_0); qb.finishApplication(app_2.getApplicationId(), user_1); - qb.releaseResource(clusterResource, app_0, - app_0.getAppSchedulingInfo().getPendingAsk(u0SchedKey) - .getPerAllocationResource(), + qb.releaseResource(clusterResource, app_0, Resource.newInstance(4*GB, 1), null, null); - qb.releaseResource(clusterResource, app_2, - app_2.getAppSchedulingInfo().getPendingAsk(u1SchedKey) - .getPerAllocationResource(), + qb.releaseResource(clusterResource, app_2, Resource.newInstance(4*GB, 1), null, null); qb.setUserLimit(50); @@ -2109,6 +2105,154 @@ public void testLocalityScheduling() throws Exception { assertEquals(numNodes+1, app_0.getSchedulingOpportunities(schedulerKey)); } + @Test + public void testRackLocalityDelayScheduling() throws Exception { + + // Change parameter values for node locality and rack locality delay. + csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 2); + csConf.setInt( + CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 1); + Map newQueues = new HashMap(); + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, + TestUtils.spyHook); + queues = newQueues; + root.reinitialize(newRoot, cs.getClusterResource()); + + // Manipulate queue 'b' + LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B)); + + // Check locality parameters. + assertEquals(2, a.getNodeLocalityDelay()); + assertEquals(1, a.getRackLocalityAdditionalDelay()); + + // User + String user1 = "user_1"; + + // Submit applications + final ApplicationAttemptId appAttemptId1 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app1 = new FiCaSchedulerApp(appAttemptId1, user1, a, + mock(ActiveUsersManager.class), spyRMContext); + a.submitApplicationAttempt(app1, user1); + + // Setup some nodes and racks + String host1 = "127.0.0.1"; + String host2 = "127.0.0.2"; + String host3 = "127.0.0.3"; + String host4 = "127.0.0.4"; + String rack1 = "rack_1"; + String rack2 = "rack_2"; + String rack3 = "rack_3"; + FiCaSchedulerNode node2 = TestUtils.getMockNode(host3, rack2, 0, 8 * GB); + FiCaSchedulerNode node3 = TestUtils.getMockNode(host4, rack3, 0, 8 * GB); + + Map apps = + ImmutableMap.of(app1.getApplicationAttemptId(), app1); + Map nodes = + ImmutableMap.of(node2.getNodeID(), node2, node3.getNodeID(), node3); + + final int numNodes = 5; + Resource clusterResource = + Resources.createResource(numNodes * (8 * GB), numNodes * 16); + when(spyRMContext.getScheduler().getNumClusterNodes()).thenReturn(numNodes); + + // Setup resource-requests and submit + Priority priority = TestUtils.createMockPriority(1); + List app1Requests1 = new ArrayList(); + app1Requests1.add(TestUtils.createResourceRequest(host1, 1 * GB, 1, + true, priority, recordFactory)); + app1Requests1.add(TestUtils.createResourceRequest(rack1, 1 * GB, 1, + true, priority, recordFactory)); + app1Requests1.add(TestUtils.createResourceRequest(host2, 1 * GB, 1, + true, priority, recordFactory)); + app1Requests1.add(TestUtils.createResourceRequest(rack2, 1 * GB, 1, + true, priority, recordFactory)); + // Adding one extra in the ANY. + app1Requests1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, + 1 * GB, 3, true, priority, recordFactory)); + app1.updateResourceRequests(app1Requests1); + + // Start testing... + CSAssignment assignment = null; + + SchedulerRequestKey schedulerKey = toSchedulerKey(priority); + assertEquals(3, app1.getOutstandingAsksCount(schedulerKey)); + + // No rack-local yet. + assignment = a.assignContainers(clusterResource, node2, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + verifyNoContainerAllocated(assignment); + assertEquals(1, app1.getSchedulingOpportunities(schedulerKey)); + assertEquals(3, app1.getOutstandingAsksCount(schedulerKey)); + assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL + + // Still no rack-local. + assignment = a.assignContainers(clusterResource, node2, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + assertEquals(2, app1.getSchedulingOpportunities(schedulerKey)); + assertEquals(3, app1.getOutstandingAsksCount(schedulerKey)); + assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL + + // Rack local now. + assignment = a.assignContainers(clusterResource, node2, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + assertEquals(0, app1.getSchedulingOpportunities(schedulerKey)); + assertEquals(2, app1.getOutstandingAsksCount(schedulerKey)); + assertEquals(NodeType.RACK_LOCAL, assignment.getType()); + + // No off-switch until 3 missed opportunities. + a.assignContainers(clusterResource, node3, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + a.assignContainers(clusterResource, node3, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + assignment = a.assignContainers(clusterResource, node3, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + assertEquals(3, app1.getSchedulingOpportunities(schedulerKey)); + assertEquals(2, app1.getOutstandingAsksCount(schedulerKey)); + assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL + + // Now off-switch should succeed. + assignment = a.assignContainers(clusterResource, node3, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + assertEquals(4, app1.getSchedulingOpportunities(schedulerKey)); + assertEquals(1, app1.getOutstandingAsksCount(schedulerKey)); + assertEquals(NodeType.OFF_SWITCH, assignment.getType()); + + // Check capping by number of cluster nodes. + doReturn(10).when(a).getRackLocalityAdditionalDelay(); + // Off-switch will happen at 6 missed opportunities now, since cluster size + // is 5. + assignment = a.assignContainers(clusterResource, node3, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + assertEquals(5, app1.getSchedulingOpportunities(schedulerKey)); + assertEquals(1, app1.getOutstandingAsksCount(schedulerKey)); + assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL + assignment = a.assignContainers(clusterResource, node3, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + applyCSAssignment(clusterResource, assignment, a, nodes, apps); + assertEquals(6, app1.getSchedulingOpportunities(schedulerKey)); + assertEquals(0, app1.getOutstandingAsksCount(schedulerKey)); + assertEquals(NodeType.OFF_SWITCH, assignment.getType()); + } + @Test public void testApplicationPriorityScheduling() throws Exception { // Manipulate queue 'a' @@ -2414,16 +2558,18 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { } @Test (timeout = 30000) - public void testNodeLocalityAfterQueueRefresh() throws Exception { + public void testLocalityDelaysAfterQueueRefresh() throws Exception { // Manipulate queue 'e' LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); // before reinitialization assertEquals(40, e.getNodeLocalityDelay()); + assertEquals(-1, e.getRackLocalityAdditionalDelay()); - csConf.setInt(CapacitySchedulerConfiguration - .NODE_LOCALITY_DELAY, 60); + csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 60); + csConf.setInt( + CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 600); Map newQueues = new HashMap(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, @@ -2435,6 +2581,7 @@ public void testNodeLocalityAfterQueueRefresh() throws Exception { // after reinitialization assertEquals(60, e.getNodeLocalityDelay()); + assertEquals(600, e.getRackLocalityAdditionalDelay()); } @Test (timeout = 30000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java index 1ea0032dc3d..9efa83d99f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ClusterNodeTracker; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; @@ -57,6 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; public class TestContinuousScheduling extends FairSchedulerTestBase { private ControlledClock mockClock; @@ -302,6 +305,39 @@ public void testThreadLifeCycle() throws InterruptedException { assertNotEquals("One of the threads is still alive", 0, numRetries); } + @Test + public void TestNodeAvailableResourceComparatorTransitivity() { + ClusterNodeTracker clusterNodeTracker = + scheduler.getNodeTracker(); + + List rmNodes = + MockNodes.newNodes(2, 4000, Resource.newInstance(4096, 4)); + for (RMNode rmNode : rmNodes) { + clusterNodeTracker.addNode(new FSSchedulerNode(rmNode, false)); + } + + // To simulate unallocated resource changes + new Thread() { + @Override + public void run() { + for (int j = 0; j < 100; j++) { + for (FSSchedulerNode node : clusterNodeTracker.getAllNodes()) { + int i = ThreadLocalRandom.current().nextInt(-30, 30); + synchronized (scheduler) { + node.deductUnallocatedResource(Resource.newInstance(i * 1024, i)); + } + } + } + } + }.start(); + + try { + scheduler.continuousSchedulingAttempt(); + } catch (Exception e) { + fail(e.getMessage()); + } + } + @Test public void testFairSchedulerContinuousSchedulingInitTime() throws Exception { scheduler.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java new file mode 100644 index 00000000000..3927b00f68e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.yarn.api.records.*; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.util.ArrayList; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test scheduler node, especially preemption reservations. + */ +public class TestFSSchedulerNode { + private final ArrayList containers = new ArrayList<>(); + + private RMNode createNode() { + RMNode node = mock(RMNode.class); + when(node.getTotalCapability()).thenReturn(Resource.newInstance(8192, 8)); + when(node.getHostName()).thenReturn("host.domain.com"); + return node; + } + + private void createDefaultContainer() { + createContainer(Resource.newInstance(1024, 1), null); + } + + private RMContainer createContainer( + Resource request, ApplicationAttemptId appAttemptId) { + RMContainer container = mock(RMContainer.class); + Container containerInner = mock(Container.class); + ContainerId id = mock(ContainerId.class); + when(id.getContainerId()).thenReturn((long)containers.size()); + when(containerInner.getResource()). + thenReturn(Resources.clone(request)); + when(containerInner.getId()).thenReturn(id); + when(containerInner.getExecutionType()). + thenReturn(ExecutionType.GUARANTEED); + when(container.getApplicationAttemptId()).thenReturn(appAttemptId); + when(container.getContainerId()).thenReturn(id); + when(container.getContainer()).thenReturn(containerInner); + when(container.getExecutionType()).thenReturn(ExecutionType.GUARANTEED); + when(container.getAllocatedResource()). + thenReturn(Resources.clone(request)); + containers.add(container); + return container; + } + + private void saturateCluster(FSSchedulerNode schedulerNode) { + while (!Resources.isNone(schedulerNode.getUnallocatedResource())) { + createDefaultContainer(); + schedulerNode.allocateContainer(containers.get(containers.size() - 1)); + schedulerNode.containerStarted(containers.get(containers.size() - 1). + getContainerId()); + } + } + + private FSAppAttempt createStarvingApp(FSSchedulerNode schedulerNode, + Resource request) { + FSAppAttempt starvingApp = mock(FSAppAttempt.class); + ApplicationAttemptId appAttemptId = + mock(ApplicationAttemptId.class); + when(starvingApp.getApplicationAttemptId()).thenReturn(appAttemptId); + when(starvingApp.assignContainer(schedulerNode)).thenAnswer( + new Answer() { + @Override + public Resource answer(InvocationOnMock invocationOnMock) + throws Throwable { + Resource response = Resource.newInstance(0, 0); + while (!Resources.isNone(request) && + !Resources.isNone(schedulerNode.getUnallocatedResource())) { + RMContainer container = createContainer(request, appAttemptId); + schedulerNode.allocateContainer(container); + Resources.addTo(response, container.getAllocatedResource()); + Resources.subtractFrom(request, + container.getAllocatedResource()); + } + return response; + } + }); + when(starvingApp.isStarved()).thenAnswer( + new Answer() { + @Override + public Boolean answer(InvocationOnMock invocationOnMock) + throws Throwable { + return !Resources.isNone(request); + } + } + ); + when(starvingApp.getPendingDemand()).thenReturn(request); + return starvingApp; + } + + private void finalValidation(FSSchedulerNode schedulerNode) { + assertEquals("Everything should have been released", + Resources.none(), schedulerNode.getAllocatedResource()); + assertTrue("No containers should be reserved for preemption", + schedulerNode.containersForPreemption.isEmpty()); + assertTrue("No resources should be reserved for preemptors", + schedulerNode.resourcesPreemptedForApp.isEmpty()); + assertEquals( + "No amount of resource should be reserved for preemptees", + Resources.none(), + schedulerNode.getTotalReserved()); + } + + private void allocateContainers(FSSchedulerNode schedulerNode) { + FairScheduler.assignPreemptedContainers(schedulerNode); + } + + /** + * Allocate and release a single container. + */ + @Test + public void testSimpleAllocation() { + RMNode node = createNode(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, false); + + createDefaultContainer(); + assertEquals("Nothing should have been allocated, yet", + Resources.none(), schedulerNode.getAllocatedResource()); + schedulerNode.allocateContainer(containers.get(0)); + assertEquals("Container should be allocated", + containers.get(0).getContainer().getResource(), + schedulerNode.getAllocatedResource()); + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + assertEquals("Everything should have been released", + Resources.none(), schedulerNode.getAllocatedResource()); + + // Check that we are error prone + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + finalValidation(schedulerNode); + } + + /** + * Allocate and release three containers with launch. + */ + @Test + public void testMultipleAllocations() { + RMNode node = createNode(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, false); + + createDefaultContainer(); + createDefaultContainer(); + createDefaultContainer(); + assertEquals("Nothing should have been allocated, yet", + Resources.none(), schedulerNode.getAllocatedResource()); + schedulerNode.allocateContainer(containers.get(0)); + schedulerNode.containerStarted(containers.get(0).getContainerId()); + schedulerNode.allocateContainer(containers.get(1)); + schedulerNode.containerStarted(containers.get(1).getContainerId()); + schedulerNode.allocateContainer(containers.get(2)); + assertEquals("Container should be allocated", + Resources.multiply(containers.get(0).getContainer().getResource(), 3.0), + schedulerNode.getAllocatedResource()); + schedulerNode.releaseContainer(containers.get(1).getContainerId(), true); + schedulerNode.releaseContainer(containers.get(2).getContainerId(), true); + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + finalValidation(schedulerNode); + } + + /** + * Allocate and release a single container. + */ + @Test + public void testSimplePreemption() { + RMNode node = createNode(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, false); + + // Launch containers and saturate the cluster + saturateCluster(schedulerNode); + assertEquals("Container should be allocated", + Resources.multiply(containers.get(0).getContainer().getResource(), + containers.size()), + schedulerNode.getAllocatedResource()); + + // Request preemption + FSAppAttempt starvingApp = createStarvingApp(schedulerNode, + Resource.newInstance(1024, 1)); + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(0)), starvingApp); + assertEquals( + "No resource amount should be reserved for preemptees", + containers.get(0).getAllocatedResource(), + schedulerNode.getTotalReserved()); + + // Preemption occurs release one container + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + allocateContainers(schedulerNode); + assertEquals("Container should be allocated", + schedulerNode.getTotalResource(), + schedulerNode.getAllocatedResource()); + + // Release all remaining containers + for (int i = 1; i < containers.size(); ++i) { + schedulerNode.releaseContainer(containers.get(i).getContainerId(), true); + } + finalValidation(schedulerNode); + } + + /** + * Allocate and release three containers requested by two apps. + */ + @Test + public void testComplexPreemption() { + RMNode node = createNode(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, false); + + // Launch containers and saturate the cluster + saturateCluster(schedulerNode); + assertEquals("Container should be allocated", + Resources.multiply(containers.get(0).getContainer().getResource(), + containers.size()), + schedulerNode.getAllocatedResource()); + + // Preempt a container + FSAppAttempt starvingApp1 = createStarvingApp(schedulerNode, + Resource.newInstance(2048, 2)); + FSAppAttempt starvingApp2 = createStarvingApp(schedulerNode, + Resource.newInstance(1024, 1)); + + // Preemption thread kicks in + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(0)), starvingApp1); + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(1)), starvingApp1); + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(2)), starvingApp2); + + // Preemption happens + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + schedulerNode.releaseContainer(containers.get(2).getContainerId(), true); + schedulerNode.releaseContainer(containers.get(1).getContainerId(), true); + + allocateContainers(schedulerNode); + assertEquals("Container should be allocated", + schedulerNode.getTotalResource(), + schedulerNode.getAllocatedResource()); + + // Release all containers + for (int i = 3; i < containers.size(); ++i) { + schedulerNode.releaseContainer(containers.get(i).getContainerId(), true); + } + finalValidation(schedulerNode); + } + + /** + * Allocate and release three containers requested by two apps in two rounds. + */ + @Test + public void testMultiplePreemptionEvents() { + RMNode node = createNode(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, false); + + // Launch containers and saturate the cluster + saturateCluster(schedulerNode); + assertEquals("Container should be allocated", + Resources.multiply(containers.get(0).getContainer().getResource(), + containers.size()), + schedulerNode.getAllocatedResource()); + + // Preempt a container + FSAppAttempt starvingApp1 = createStarvingApp(schedulerNode, + Resource.newInstance(2048, 2)); + FSAppAttempt starvingApp2 = createStarvingApp(schedulerNode, + Resource.newInstance(1024, 1)); + + // Preemption thread kicks in + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(0)), starvingApp1); + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(1)), starvingApp1); + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(2)), starvingApp2); + + // Preemption happens + schedulerNode.releaseContainer(containers.get(1).getContainerId(), true); + allocateContainers(schedulerNode); + + schedulerNode.releaseContainer(containers.get(2).getContainerId(), true); + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + allocateContainers(schedulerNode); + + assertEquals("Container should be allocated", + schedulerNode.getTotalResource(), + schedulerNode.getAllocatedResource()); + + // Release all containers + for (int i = 3; i < containers.size(); ++i) { + schedulerNode.releaseContainer(containers.get(i).getContainerId(), true); + } + finalValidation(schedulerNode); + } + + /** + * Allocate and release a single container and delete the app in between. + */ + @Test + public void testPreemptionToCompletedApp() { + RMNode node = createNode(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, false); + + // Launch containers and saturate the cluster + saturateCluster(schedulerNode); + assertEquals("Container should be allocated", + Resources.multiply(containers.get(0).getContainer().getResource(), + containers.size()), + schedulerNode.getAllocatedResource()); + + // Preempt a container + FSAppAttempt starvingApp = createStarvingApp(schedulerNode, + Resource.newInstance(1024, 1)); + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(0)), starvingApp); + + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + + // Stop the application then try to satisfy the reservation + // and observe that there are still free resources not allocated to + // the deleted app + when(starvingApp.isStopped()).thenReturn(true); + allocateContainers(schedulerNode); + assertNotEquals("Container should be allocated", + schedulerNode.getTotalResource(), + schedulerNode.getAllocatedResource()); + + // Release all containers + for (int i = 1; i < containers.size(); ++i) { + schedulerNode.releaseContainer(containers.get(i).getContainerId(), true); + } + finalValidation(schedulerNode); + } + + /** + * Preempt a bigger container than the preemption request. + */ + @Test + public void testPartialReservedPreemption() { + RMNode node = createNode(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, false); + + // Launch containers and saturate the cluster + saturateCluster(schedulerNode); + assertEquals("Container should be allocated", + Resources.multiply(containers.get(0).getContainer().getResource(), + containers.size()), + schedulerNode.getAllocatedResource()); + + // Preempt a container + Resource originalStarvingAppDemand = Resource.newInstance(512, 1); + FSAppAttempt starvingApp = createStarvingApp(schedulerNode, + originalStarvingAppDemand); + schedulerNode.addContainersForPreemption( + Collections.singletonList(containers.get(0)), starvingApp); + + // Preemption occurs + schedulerNode.releaseContainer(containers.get(0).getContainerId(), true); + + // Container partially reassigned + allocateContainers(schedulerNode); + assertEquals("Container should be allocated", + Resources.subtract(schedulerNode.getTotalResource(), + Resource.newInstance(512, 0)), + schedulerNode.getAllocatedResource()); + + // Cleanup simulating node update + schedulerNode.getPreemptionList(); + + // Release all containers + for (int i = 1; i < containers.size(); ++i) { + schedulerNode.releaseContainer(containers.get(i).getContainerId(), true); + } + finalValidation(schedulerNode); + } + +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java index 3940a47aded..59d243bc690 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java @@ -294,11 +294,30 @@ private void verifyPreemption(int numStarvedAppContainers) 8 - 2 * numStarvedAppContainers, greedyApp.getQueue().getMetrics().getAggregatePreemptedContainers()); + // Verify the node is reserved for the starvingApp + for (RMNode rmNode : rmNodes) { + FSSchedulerNode node = (FSSchedulerNode) + scheduler.getNodeTracker().getNode(rmNode.getNodeID()); + if (node.getContainersForPreemption().size() > 0) { + assertTrue("node should be reserved for the starvingApp", + node.getPreemptionList().keySet().contains(starvingApp)); + } + } + sendEnoughNodeUpdatesToAssignFully(); // Verify the preempted containers are assigned to starvingApp assertEquals("Starved app is not assigned the right # of containers", numStarvedAppContainers, starvingApp.getLiveContainers().size()); + + // Verify the node is not reserved for the starvingApp anymore + for (RMNode rmNode : rmNodes) { + FSSchedulerNode node = (FSSchedulerNode) + scheduler.getNodeTracker().getNode(rmNode.getNodeID()); + if (node.getContainersForPreemption().size() > 0) { + assertFalse(node.getPreemptionList().keySet().contains(starvingApp)); + } + } } private void verifyNoPreemption() throws InterruptedException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java index 8dccf6e6130..d84f0cf73e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java @@ -55,7 +55,6 @@ public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); } - @Test(timeout = 1000) public void testParseSchedulingPolicy() throws AllocationConfigurationException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index fb9e8edfb0e..aab9bee766b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -24,6 +24,7 @@ import static org.junit.Assert.fail; import java.io.StringReader; +import java.util.ArrayList; import java.util.Collection; import javax.ws.rs.core.MediaType; @@ -46,6 +47,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -169,7 +172,38 @@ public void testAppsXML() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, nodesApps.getLength()); NodeList nodes = dom.getElementsByTagName("app"); assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1); + verifyAppsXML(nodes, app1, false); + rm.stop(); + } + + @Test + public void testRunningApp() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); + am1.allocate("*", 4096, 1, new ArrayList<>()); + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodesApps = dom.getElementsByTagName("apps"); + assertEquals("incorrect number of elements", 1, nodesApps.getLength()); + NodeList nodes = dom.getElementsByTagName("app"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + verifyAppsXML(nodes, app1, true); + + testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON, true); rm.stop(); } @@ -203,6 +237,11 @@ public void testAppsXMLMulti() throws JSONException, Exception { public void testAppsHelper(String path, RMApp app, String media) throws JSONException, Exception { + testAppsHelper(path, app, media, false); + } + + public void testAppsHelper(String path, RMApp app, String media, + boolean hasResourceReq) throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -215,7 +254,7 @@ public void testAppsHelper(String path, RMApp app, String media) assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app); + verifyAppInfo(array.getJSONObject(0), app, hasResourceReq); } @@ -239,7 +278,7 @@ public void testAppsQueryState() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app1); + verifyAppInfo(array.getJSONObject(0), app1, false); rm.stop(); } @@ -483,7 +522,7 @@ public void testAppsQueryFinalStatus() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app1); + verifyAppInfo(array.getJSONObject(0), app1, false); rm.stop(); } @@ -1327,7 +1366,7 @@ public void testSingleAppsHelper(String path, RMApp app, String media) JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - verifyAppInfo(json.getJSONObject("app"), app); + verifyAppInfo(json.getJSONObject("app"), app, false); } @Test @@ -1351,11 +1390,11 @@ public void testSingleAppsXML() throws JSONException, Exception { Document dom = db.parse(is); NodeList nodes = dom.getElementsByTagName("app"); assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1); + verifyAppsXML(nodes, app1, false); rm.stop(); } - public void verifyAppsXML(NodeList nodes, RMApp app) + public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) throws JSONException, Exception { for (int i = 0; i < nodes.getLength(); i++) { @@ -1394,32 +1433,38 @@ public void verifyAppsXML(NodeList nodes, RMApp app) WebServicesTestUtils.getXmlString(element, "amNodeLabelExpression"), WebServicesTestUtils.getXmlString(element, "amRPCAddress")); - assertEquals(element.getElementsByTagName("resourceRequests").getLength(), - 1); - Element resourceRequests = - (Element) element.getElementsByTagName("resourceRequests").item(0); - Element capability = - (Element) resourceRequests.getElementsByTagName("capability").item(0); - - verifyResourceRequestsGeneric(app, - WebServicesTestUtils.getXmlString(resourceRequests, - "nodeLabelExpression"), - WebServicesTestUtils.getXmlInt(resourceRequests, "numContainers"), - WebServicesTestUtils.getXmlBoolean(resourceRequests, "relaxLocality"), - WebServicesTestUtils.getXmlInt(resourceRequests, "priority"), - WebServicesTestUtils.getXmlString(resourceRequests, "resourceName"), - WebServicesTestUtils.getXmlLong(capability, "memory"), - WebServicesTestUtils.getXmlLong(capability, "vCores"), - WebServicesTestUtils.getXmlString(resourceRequests, "executionType"), - WebServicesTestUtils.getXmlBoolean(resourceRequests, - "enforceExecutionType")); + if (hasResourceReq) { + assertEquals(element.getElementsByTagName("resourceRequests").getLength(), + 1); + Element resourceRequests = + (Element) element.getElementsByTagName("resourceRequests").item(0); + Element capability = + (Element) resourceRequests.getElementsByTagName("capability").item(0); + ResourceRequest rr = + ((AbstractYarnScheduler)rm.getRMContext().getScheduler()) + .getApplicationAttempt( + app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + verifyResourceRequestsGeneric(rr, + WebServicesTestUtils.getXmlString(resourceRequests, + "nodeLabelExpression"), + WebServicesTestUtils.getXmlInt(resourceRequests, "numContainers"), + WebServicesTestUtils.getXmlBoolean(resourceRequests, "relaxLocality"), + WebServicesTestUtils.getXmlInt(resourceRequests, "priority"), + WebServicesTestUtils.getXmlString(resourceRequests, "resourceName"), + WebServicesTestUtils.getXmlLong(capability, "memory"), + WebServicesTestUtils.getXmlLong(capability, "vCores"), + WebServicesTestUtils.getXmlString(resourceRequests, "executionType"), + WebServicesTestUtils.getXmlBoolean(resourceRequests, + "enforceExecutionType")); + } } } - public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, - Exception { + public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) + throws JSONException, Exception { - int expectedNumberOfElements = 35; + int expectedNumberOfElements = 34 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() @@ -1461,7 +1506,9 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, amNodeLabelExpression, amRPCAddress); - verifyResourceRequests(info.getJSONArray("resourceRequests"), app); + if (hasResourceReqs) { + verifyResourceRequests(info.getJSONArray("resourceRequests"), app); + } } public void verifyAppInfoGeneric(RMApp app, String id, String user, @@ -1490,8 +1537,10 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, WebServicesTestUtils.checkStringMatch("finalStatus", app .getFinalApplicationStatus().toString(), finalStatus); assertEquals("progress doesn't match", 0, progress, 0.0); - WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED", - trackingUI); + if ("UNASSIGNED".equals(trackingUI)) { + WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED", + trackingUI); + } WebServicesTestUtils.checkStringEqual("diagnostics", app.getDiagnostics().toString(), diagnostics); assertEquals("clusterId doesn't match", @@ -1544,7 +1593,12 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, public void verifyResourceRequests(JSONArray resourceRequest, RMApp app) throws JSONException { JSONObject requestInfo = resourceRequest.getJSONObject(0); - verifyResourceRequestsGeneric(app, + ResourceRequest rr = + ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) + .getApplicationAttempt( + app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + verifyResourceRequestsGeneric(rr, requestInfo.getString("nodeLabelExpression"), requestInfo.getInt("numContainers"), requestInfo.getBoolean("relaxLocality"), requestInfo.getInt("priority"), @@ -1557,11 +1611,10 @@ public void verifyResourceRequests(JSONArray resourceRequest, RMApp app) .getBoolean("enforceExecutionType")); } - public void verifyResourceRequestsGeneric(RMApp app, + public void verifyResourceRequestsGeneric(ResourceRequest request, String nodeLabelExpression, int numContainers, boolean relaxLocality, int priority, String resourceName, long memory, long vCores, String executionType, boolean enforceExecutionType) { - ResourceRequest request = app.getAMResourceRequests().get(0); assertEquals("nodeLabelExpression doesn't match", request.getNodeLabelExpression(), nodeLabelExpression); assertEquals("numContainers doesn't match", request.getNumContainers(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 77240c63f64..974ef093c16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -146,7 +146,6 @@ public class MiniYARNCluster extends CompositeService { private int numLocalDirs; // Number of nm-log-dirs per nodemanager private int numLogDirs; - private boolean enableAHS; /** * @param testName name of the test @@ -154,16 +153,13 @@ public class MiniYARNCluster extends CompositeService { * @param numNodeManagers the number of node managers in the cluster * @param numLocalDirs the number of nm-local-dirs per nodemanager * @param numLogDirs the number of nm-log-dirs per nodemanager - * @param enableAHS enable ApplicationHistoryServer or not */ - @Deprecated public MiniYARNCluster( String testName, int numResourceManagers, int numNodeManagers, - int numLocalDirs, int numLogDirs, boolean enableAHS) { + int numLocalDirs, int numLogDirs) { super(testName.replace("$", "")); this.numLocalDirs = numLocalDirs; this.numLogDirs = numLogDirs; - this.enableAHS = enableAHS; String testSubDir = testName.replace("$", ""); File targetWorkDir = new File("target", testSubDir); try { @@ -213,20 +209,6 @@ public MiniYARNCluster( nodeManagers = new NodeManager[numNodeManagers]; } - /** - * @param testName name of the test - * @param numResourceManagers the number of resource managers in the cluster - * @param numNodeManagers the number of node managers in the cluster - * @param numLocalDirs the number of nm-local-dirs per nodemanager - * @param numLogDirs the number of nm-log-dirs per nodemanager - */ - public MiniYARNCluster( - String testName, int numResourceManagers, int numNodeManagers, - int numLocalDirs, int numLogDirs) { - this(testName, numResourceManagers, numNodeManagers, numLocalDirs, - numLogDirs, false); - } - /** * @param testName name of the test * @param numNodeManagers the number of node managers in the cluster @@ -288,7 +270,7 @@ public void serviceInit(Configuration conf) throws Exception { } if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) { + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { addService(new ApplicationHistoryServerWrapper()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java index ff7fafc2001..92f63eaa580 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java @@ -34,18 +34,14 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { int numNodeManagers = 1; int numLocalDirs = 1; int numLogDirs = 1; - boolean enableAHS; /* * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false - * and enableAHS flag == false */ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); - enableAHS = false; try (MiniYARNCluster cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, - enableAHS)) { + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs)) { cluster.init(conf); cluster.start(); @@ -57,14 +53,11 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { /* * Timeline service should start if TIMELINE_SERVICE_ENABLED == true - * and enableAHS == false */ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); - enableAHS = false; try (MiniYARNCluster cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, - enableAHS)) { + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs)) { cluster.init(conf); // Verify that the timeline-service starts on ephemeral ports by default @@ -74,29 +67,6 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { cluster.start(); - //Timeline service may sometime take a while to get started - int wait = 0; - while(cluster.getApplicationHistoryServer() == null && wait < 20) { - Thread.sleep(500); - wait++; - } - //verify that the timeline service is started. - Assert.assertNotNull("Timeline Service should have been started", - cluster.getApplicationHistoryServer()); - } - /* - * Timeline service should start if TIMELINE_SERVICE_ENABLED == false - * and enableAHS == true - */ - conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); - enableAHS = true; - try (MiniYARNCluster cluster = - new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, - enableAHS)) { - cluster.init(conf); - cluster.start(); - //Timeline service may sometime take a while to get started int wait = 0; while(cluster.getApplicationHistoryServer() == null && wait < 20) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml index 4dc849277e3..a3a44f81c6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml @@ -108,6 +108,11 @@ hadoop-hdfs test + + org.apache.hadoop + hadoop-hdfs-client + test + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index e66d079363f..4de0a6ae489 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -269,8 +269,8 @@ To submit the pi job to run in Docker containers, run the following commands: ``` vars="YARN_CONTAINER_RUNTIME_TYPE=docker,YARN_CONTAINER_RUNTIME_DOCKER_IMAGE=hadoop-docker" - hadoop jar hadoop-examples.jar -Dyarn.app.mapreduce.am.env=$vars \ - -Dmapreduce.map.env=$vars -Dmapreduce.reduce.env=$vars pi 10 100 + hadoop jar hadoop-examples.jar pi -Dyarn.app.mapreduce.am.env=$vars \ + -Dmapreduce.map.env=$vars -Dmapreduce.reduce.env=$vars 10 100 ``` Note that the application master, map tasks, and reduce tasks are configured diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml index d42acaac751..5ff012baa17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml @@ -123,7 +123,7 @@ com.github.eirslett frontend-maven-plugin - 0.0.22 + 1.1 ${webappTgtDir} @@ -172,7 +172,7 @@ ${webappTgtDir} ${node.executable} - node/npm/bin/npm-cli + node/node_modules/npm/bin/npm-cli run build:mvn diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-attempt-table.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-attempt-table.js index 3c430376dd1..88282750e23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-attempt-table.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/app-attempt-table.js @@ -19,11 +19,4 @@ import Ember from 'ember'; export default Ember.Component.extend({ - nodeHttpAddressFormatted: Ember.computed('attempt.nodeHttpAddress', function() { - var nodeHttpAddress = this.get('attempt.nodeHttpAddress'); - if (nodeHttpAddress && nodeHttpAddress.indexOf('://') < 0) { - nodeHttpAddress = 'http://' + nodeHttpAddress; - } - return nodeHttpAddress; - }) }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js index d730a43c436..4a33d5b6d94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js @@ -18,6 +18,7 @@ import Ember from 'ember'; import Converter from 'yarn-ui/utils/converter'; +import ColumnDef from 'em-table/utils/column-definition'; export default Ember.Component.extend({ canvas: { @@ -31,6 +32,8 @@ export default Ember.Component.extend({ modelArr: [], colors: d3.scale.category10().range(), _selected: undefined, + gridColumns: [], + gridRows: [], selected: function() { return this._selected; @@ -276,5 +279,199 @@ export default Ember.Component.extend({ if (this.modelArr.length > 0) { this.setSelected(this.modelArr[0]); } + + if (this.get('attemptModel')) { + this.setAttemptsGridColumnsAndRows(); + } else { + this.setContainersGridColumnsAndRows(); + } }, -}); \ No newline at end of file + + setAttemptsGridColumnsAndRows: function() { + var self = this; + var columns = []; + + columns.push({ + id: 'id', + headerTitle: 'Attempt ID', + contentPath: 'id', + cellComponentName: 'em-table-linked-cell', + minWidth: '300px', + getCellContent: function(row) { + return { + displayText: row.get('id'), + routeName: 'yarn-app-attempt', + id: row.get('id') + }; + } + }, { + id: 'attemptStartedTime', + headerTitle: 'Started Time', + contentPath: 'attemptStartedTime' + }, { + id: 'finishedTime', + headerTitle: 'Finished Time', + contentPath: 'finishedTime', + getCellContent: function(row) { + if (row.get('finishedTs')) { + return row.get('finishedTime'); + } + return 'N/A'; + } + }, { + id: 'elapsedTime', + headerTitle: 'Elapsed Time', + contentPath: 'elapsedTime' + }, { + id: 'appMasterContainerId', + headerTitle: 'AM Container ID', + contentPath: 'appMasterContainerId', + minWidth: '300px' + }, { + id: 'amNodeId', + headerTitle: 'AM Node ID', + contentPath: 'amNodeId' + }, { + id: 'attemptState', + headerTitle: 'State', + contentPath: 'attemptState', + getCellContent: function(row) { + var state = row.get('attemptState'); + if (state) { + return state; + } else { + return 'N/A'; + } + } + }, { + id: 'nodeHttpAddress', + headerTitle: 'NodeManager Web UI', + contentPath: 'nodeHttpAddress', + cellComponentName: 'em-table-html-cell', + getCellContent: function(row) { + var address = self.checkHttpProtocol(row.get('nodeHttpAddress')); + if (address) { + return `${address}`; + } else { + return 'N/A'; + } + } + }, { + id: 'logsLink', + headerTitle: 'Logs', + contentPath: 'logsLink', + cellComponentName: 'em-table-html-cell', + getCellContent: function(row) { + var logUrl = self.checkHttpProtocol(row.get('logsLink')); + if (logUrl) { + return `Link`; + } else { + return 'N/A'; + } + } + }); + + var gridCols = ColumnDef.make(columns); + this.set('gridColumns', gridCols); + this.set('gridRows', this.modelArr); + }, + + setContainersGridColumnsAndRows: function() { + var self = this; + var columns = []; + + columns.push({ + id: 'id', + headerTitle: 'Container ID', + contentPath: 'id', + minWidth: '300px' + }, { + id: 'startedTime', + headerTitle: 'Started Time', + contentPath: 'startedTime' + }, { + id: 'finishedTime', + headerTitle: 'Finished Time', + contentPath: 'finishedTime', + getCellContent: function(row) { + if (row.get('finishedTs')) { + return row.get('finishedTime'); + } + return 'N/A'; + } + }, { + id: 'elapsedTime', + headerTitle: 'Elapsed Time', + contentPath: 'elapsedTime' + }, { + id: 'priority', + headerTitle: 'Priority', + contentPath: 'priority' + }, { + id: 'containerExitStatus', + headerTitle: 'Exit Status', + contentPath: 'containerExitStatus', + getCellContent: function(row) { + var status = row.get('containerExitStatus'); + if (status) { + return status; + } else { + return 'N/A'; + } + } + }, { + id: 'containerState', + headerTitle: 'State', + contentPath: 'containerState', + getCellContent: function(row) { + var state = row.get('containerState'); + if (state) { + return state; + } else { + return 'N/A'; + } + } + }, { + id: 'logUrl', + headerTitle: 'Logs', + contentPath: 'logUrl', + cellComponentName: 'em-table-html-cell', + getCellContent: function(row) { + var url = self.checkHttpProtocol(row.get('logUrl')); + if (url) { + return `${url}`; + } else { + return 'N/A'; + } + } + }, { + id: 'nodeHttpAddress', + headerTitle: 'Node Manager UI', + contentPath: 'nodeHttpAddress', + cellComponentName: 'em-table-html-cell', + getCellContent: function(row) { + var address = self.checkHttpProtocol(row.get('nodeHttpAddress')); + if (address) { + return `${address}`; + } else { + return 'N/A'; + } + } + }); + + var gridCols = ColumnDef.make(columns); + this.set('gridColumns', gridCols); + this.set('gridRows', this.modelArr); + }, + + checkHttpProtocol: function(prop) { + if (prop && prop.indexOf('://') < 0) { + prop = 'http://' + prop; + } + return prop; + }, + + isDataEmpty: Ember.computed(function() { + return this.modelArr.length === 0; + }) +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/prepend-protocol.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/prepend-protocol.js new file mode 100644 index 00000000000..e8d18c44720 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/prepend-protocol.js @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; + +export function prependProtocol(params/*, hash*/) { + let address = params[0]; + if (address && address.indexOf('://') < 0) { + address = 'http://' + address; + } + return address; +} + +export default Ember.Helper.helper(prependProtocol); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js index 19863e1c8d1..807844e2b22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js @@ -61,7 +61,11 @@ export default Ember.Service.extend({ }, localBaseAddress: Ember.computed(function () { - return this.localAddress(); + var url = this.localAddress(); + if (url.endsWith('/')) { + url = url.slice(0, -1); + } + return url; }), timelineWebAddress: Ember.computed(function () { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-attempt-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-attempt-table.hbs index fcd076b2ebe..c3a9e32e77d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-attempt-table.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-attempt-table.hbs @@ -23,39 +23,43 @@ {{attempt.id}} - Start Time + Started Time {{attempt.attemptStartedTime}} + {{#if attempt.validatedFinishedTs}} + + Finished Time + {{attempt.validatedFinishedTs}} + + {{/if}} + + Elapsed Time + {{attempt.elapsedTime}} + AM Container Id {{attempt.appMasterContainerId}} - {{#if attempt.IsAmNodeUrl}} - - AM Node Web UI - {{nodeHttpAddressFormatted}} - - {{/if}} AM Node Id {{attempt.amNodeId}} - {{#if attempt.IsLinkAvailable}} - - Log - Link - - {{/if}} {{#if attempt.attemptState}} Attempt State {{attempt.attemptState}} {{/if}} - {{#if attempt.elapsedTime}} + {{#if attempt.nodeHttpAddress}} - Elapsed Time - {{attempt.elapsedTime}} + AM Node Web UI + {{attempt.nodeHttpAddress}} + + {{/if}} + {{#if attempt.logsLink}} + + Log + Link {{/if}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/container-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/container-table.hbs index 586f128613e..ab353d25d52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/container-table.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/container-table.hbs @@ -19,13 +19,15 @@ - + + {{#if container.validatedFinishedTs}} + {{/if}} @@ -34,21 +36,29 @@ - - - - + {{#if container.containerExitStatus}} + {{/if}} + {{#if container.containerState}} + {{/if}} + {{#if container.nodeHttpAddress}} - + + {{/if}} + {{#if container.logUrl}} + + + + + {{/if}}
Start TimeStarted Time {{container.startedTime}}
Finished Time {{container.validatedFinishedTs}}
Elapsed Time {{container.elapsedTime}}Priority {{container.priority}}
LogLink
Exit Status {{container.containerExitStatus}}
State {{container.containerState}}
NodeManager UI{{container.nodeHttpAddress}}{{container.nodeHttpAddress}}
LogLink
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/timeline-view.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/timeline-view.hbs index b110268731d..78561948f1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/timeline-view.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/timeline-view.hbs @@ -25,30 +25,49 @@ Containers {{/if}} -
-

-
-
- - -
-
-
- {{#if selected.link}} - {{#link-to selected.linkname selected.id}}{{selected.id}}{{/link-to}} - {{else}} - {{selected.id}} - {{/if}} + {{#if isDataEmpty}} + +
+
+
+

+
+ +
+
+
+ {{#if selected.link}} + {{#link-to selected.linkname selected.id}}{{selected.id}}{{/link-to}} + {{else}} + {{selected.id}} + {{/if}} +
+ {{#if attemptModel}} + {{app-attempt-table attempt=selected}} + {{else}} + {{container-table container=selected}} + {{/if}} +
+
+
+
+ {{em-table columns=gridColumns rows=gridRows}}
- {{#if attemptModel}} - {{app-attempt-table attempt=selected}} - {{else}} - {{container-table container=selected}} - {{/if}}
-
+ {{else}} +
+

No data available!

+
+ {{/if}}
-{{outlet}} \ No newline at end of file +{{outlet}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/helpers/prepend-protocol-test.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/helpers/prepend-protocol-test.js new file mode 100644 index 00000000000..6dc8137c26f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/helpers/prepend-protocol-test.js @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { prependProtocol } from '../../../helpers/prepend-protocol'; +import { module, test } from 'qunit'; + +module('Unit | Helper | prepend protocol'); + +// Replace this with your real tests. +test('it works', function(assert) { + let result = prependProtocol(42); + assert.ok(result); +});