From 9833468302bd2fa235d9d1f40517631f9dfff517 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Fri, 20 Jul 2012 18:58:58 +0000 Subject: [PATCH 01/39] HDFS-3597. SNN fails to start after DFS upgrade. Contributed by Andy Isaacson. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1363899 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../server/namenode/CheckpointSignature.java | 17 ++- .../server/namenode/SecondaryNameNode.java | 16 ++- .../TestSecondaryNameNodeUpgrade.java | 116 ++++++++++++++++++ 4 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7d108d9de1a..cc16ac35638 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -518,6 +518,8 @@ Branch-2 ( Unreleased changes ) HDFS-3690. BlockPlacementPolicyDefault incorrectly casts LOG. (eli) + HDFS-3597. SNN fails to start after DFS upgrade. (Andy Isaacson via todd) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java index d723ede9bf1..8c4d79cc9bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java @@ -113,12 +113,19 @@ public class CheckpointSignature extends StorageInfo + blockpoolID ; } + boolean storageVersionMatches(StorageInfo si) throws IOException { + return (layoutVersion == si.layoutVersion) && (cTime == si.cTime); + } + + boolean isSameCluster(FSImage si) { + return namespaceID == si.getStorage().namespaceID && + clusterID.equals(si.getClusterID()) && + blockpoolID.equals(si.getBlockPoolID()); + } + void validateStorageInfo(FSImage si) throws IOException { - if(layoutVersion != si.getStorage().layoutVersion - || namespaceID != si.getStorage().namespaceID - || cTime != si.getStorage().cTime - || !clusterID.equals(si.getClusterID()) - || !blockpoolID.equals(si.getBlockPoolID())) { + if (!isSameCluster(si) + || !storageVersionMatches(si.getStorage())) { throw new IOException("Inconsistent checkpoint fields.\n" + "LV = " + layoutVersion + " namespaceID = " + namespaceID + " cTime = " + cTime diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index c32bd3383b7..8057955dfac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -437,18 +437,16 @@ public class SecondaryNameNode implements Runnable { // Returns a token that would be used to upload the merged image. CheckpointSignature sig = namenode.rollEditLog(); - // Make sure we're talking to the same NN! - if (checkpointImage.getNamespaceID() != 0) { - // If the image actually has some data, make sure we're talking - // to the same NN as we did before. - sig.validateStorageInfo(checkpointImage); - } else { - // if we're a fresh 2NN, just take the storage info from the server - // we first talk to. + if ((checkpointImage.getNamespaceID() == 0) || + (sig.isSameCluster(checkpointImage) && + !sig.storageVersionMatches(checkpointImage.getStorage()))) { + // if we're a fresh 2NN, or if we're on the same cluster and our storage + // needs an upgrade, just take the storage info from the server. dstStorage.setStorageInfo(sig); dstStorage.setClusterID(sig.getClusterID()); dstStorage.setBlockPoolID(sig.getBlockpoolID()); } + sig.validateStorageInfo(checkpointImage); // error simulation code for junit test CheckpointFaultInjector.getInstance().afterSecondaryCallsRollEditLog(); @@ -703,7 +701,7 @@ public class SecondaryNameNode implements Runnable { /** * Analyze checkpoint directories. * Create directories if they do not exist. - * Recover from an unsuccessful checkpoint is necessary. + * Recover from an unsuccessful checkpoint if necessary. * * @throws IOException */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java new file mode 100644 index 00000000000..6119584c0d8 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.junit.Test; +import org.junit.Before; +import org.junit.After; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileUtil; + +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; + +import java.util.Properties; +import java.io.FileReader; +import java.io.FileWriter; +import org.junit.Assert; +import org.apache.hadoop.test.GenericTestUtils; + +/** + * Regression test for HDFS-3597, SecondaryNameNode upgrade -- when a 2NN + * starts up with an existing directory structure with an old VERSION file, it + * should delete the snapshot and download a new one from the NN. + */ +public class TestSecondaryNameNodeUpgrade { + + @Before + public void cleanupCluster() throws IOException { + File hdfsDir = new File(MiniDFSCluster.getBaseDirectory()).getCanonicalFile(); + System.out.println("cleanupCluster deleting " + hdfsDir); + if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) { + throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'"); + } + } + + private void doIt(String param, String val) throws IOException { + MiniDFSCluster cluster = null; + FileSystem fs = null; + SecondaryNameNode snn = null; + + try { + Configuration conf = new HdfsConfiguration(); + + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + + conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0"); + snn = new SecondaryNameNode(conf); + + fs = cluster.getFileSystem(); + + fs.mkdirs(new Path("/test/foo")); + + snn.doCheckpoint(); + + List versionFiles = snn.getFSImage().getStorage().getFiles(null, "VERSION"); + + snn.shutdown(); + + for (File versionFile : versionFiles) { + System.out.println("Changing '" + param + "' to '" + val + "' in " + versionFile); + FSImageTestUtil.corruptVersionFile(versionFile, param, val); + } + + snn = new SecondaryNameNode(conf); + + fs.mkdirs(new Path("/test/bar")); + + snn.doCheckpoint(); + } finally { + if (fs != null) fs.close(); + if (cluster != null) cluster.shutdown(); + if (snn != null) snn.shutdown(); + } + } + + @Test + public void testUpgradeLayoutVersionSucceeds() throws IOException { + doIt("layoutVersion", "-39"); + } + + @Test + public void testChangeNsIDFails() throws IOException { + try { + doIt("namespaceID", "2"); + Assert.fail("Should throw InconsistentFSStateException"); + } catch(IOException e) { + GenericTestUtils.assertExceptionContains("Inconsistent checkpoint fields", e); + System.out.println("Correctly failed with inconsistent namespaceID: " + e); + } + } +} From 02a5551ba3d3d44b79bf594816263d625cdca36f Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 20 Jul 2012 19:15:52 +0000 Subject: [PATCH 02/39] HDFS-3608. fuse_dfs: detect changes in UID ticket cache. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1363904 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop-hdfs/src/CMakeLists.txt | 1 + .../src/main/native/fuse-dfs/CMakeLists.txt | 1 + .../src/main/native/fuse-dfs/fuse_connect.c | 638 ++++++++++++--- .../src/main/native/fuse-dfs/fuse_connect.h | 70 +- .../native/fuse-dfs/fuse_context_handle.h | 2 - .../src/main/native/fuse-dfs/fuse_dfs.c | 37 +- .../main/native/fuse-dfs/fuse_file_handle.h | 4 +- .../main/native/fuse-dfs/fuse_impls_chmod.c | 16 +- .../main/native/fuse-dfs/fuse_impls_chown.c | 28 +- .../main/native/fuse-dfs/fuse_impls_flush.c | 5 +- .../main/native/fuse-dfs/fuse_impls_getattr.c | 28 +- .../main/native/fuse-dfs/fuse_impls_mkdir.c | 28 +- .../main/native/fuse-dfs/fuse_impls_open.c | 65 +- .../main/native/fuse-dfs/fuse_impls_read.c | 9 +- .../main/native/fuse-dfs/fuse_impls_readdir.c | 26 +- .../main/native/fuse-dfs/fuse_impls_release.c | 6 +- .../main/native/fuse-dfs/fuse_impls_rename.c | 29 +- .../main/native/fuse-dfs/fuse_impls_rmdir.c | 46 +- .../main/native/fuse-dfs/fuse_impls_statfs.c | 35 +- .../native/fuse-dfs/fuse_impls_truncate.c | 22 +- .../main/native/fuse-dfs/fuse_impls_unlink.c | 31 +- .../main/native/fuse-dfs/fuse_impls_utimens.c | 25 +- .../main/native/fuse-dfs/fuse_impls_write.c | 7 +- .../src/main/native/fuse-dfs/fuse_init.c | 26 +- .../src/main/native/libhdfs/hdfs.c | 72 +- .../src/main/native/libhdfs/hdfs.h | 24 +- .../src/main/native/libhdfs/jni_helper.c | 6 + .../hadoop-hdfs/src/main/native/util/tree.h | 765 ++++++++++++++++++ .../src/main/resources/hdfs-default.xml | 21 + 30 files changed, 1740 insertions(+), 336 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/tree.h diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cc16ac35638..7a2c6acf994 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -520,6 +520,9 @@ Branch-2 ( Unreleased changes ) HDFS-3597. SNN fails to start after DFS upgrade. (Andy Isaacson via todd) + HDFS-3608. fuse_dfs: detect changes in UID ticket cache. (Colin Patrick + McCabe via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt index 4a96bbd7a0f..7c1441fca52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt @@ -87,6 +87,7 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR} ${JNI_INCLUDE_DIRS} + main/native main/native/libhdfs ) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt index fb3c580e94c..f04870957e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt @@ -69,5 +69,6 @@ IF(FUSE_FOUND) ${JAVA_JVM_LIBRARY} hdfs m + pthread ) ENDIF(FUSE_FOUND) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c index bfb7a1eedb7..c6624fad999 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c @@ -16,17 +16,38 @@ * limitations under the License. */ -#include "hdfs.h" -#include "fuse_dfs.h" #include "fuse_connect.h" +#include "fuse_dfs.h" #include "fuse_users.h" +#include "libhdfs/hdfs.h" +#include "util/tree.h" +#include #include +#include #include #include #include +#include +#include +#include -#define HADOOP_SECURITY_AUTHENTICATION "hadoop.security.authentication" +#define FUSE_CONN_DEFAULT_TIMER_PERIOD 5 +#define FUSE_CONN_DEFAULT_EXPIRY_PERIOD (5 * 60) +#define HADOOP_SECURITY_AUTHENTICATION "hadoop.security.authentication" +#define HADOOP_FUSE_CONNECTION_TIMEOUT "hadoop.fuse.connection.timeout" +#define HADOOP_FUSE_TIMER_PERIOD "hadoop.fuse.timer.period" + +/** Length of the buffer needed by asctime_r */ +#define TIME_STR_LEN 26 + +struct hdfsConn; + +static int hdfsConnCompare(const struct hdfsConn *a, const struct hdfsConn *b); +static void hdfsConnExpiry(void); +static void* hdfsConnExpiryThread(void *v); + +RB_HEAD(hdfsConnTree, hdfsConn); enum authConf { AUTH_CONF_UNKNOWN, @@ -34,58 +55,54 @@ enum authConf { AUTH_CONF_OTHER, }; -#define MAX_ELEMENTS (16 * 1024) -static struct hsearch_data *fsTable = NULL; -static enum authConf hdfsAuthConf = AUTH_CONF_UNKNOWN; -static pthread_mutex_t tableMutex = PTHREAD_MUTEX_INITIALIZER; +struct hdfsConn { + RB_ENTRY(hdfsConn) entry; + /** How many threads are currently using this hdfsConnection object */ + int64_t refcnt; + /** The username used to make this connection. Dynamically allocated. */ + char *usrname; + /** Kerberos ticket cache path, or NULL if this is not a kerberized + * connection. Dynamically allocated. */ + char *kpath; + /** mtime of the kpath, if the kpath is non-NULL */ + time_t kPathMtime; + /** nanosecond component of the mtime of the kpath, if the kpath is non-NULL */ + long kPathMtimeNs; + /** The cached libhdfs fs instance */ + hdfsFS fs; + /** Nonzero if this hdfs connection needs to be closed as soon as possible. + * If this is true, the connection has been removed from the tree. */ + int condemned; + /** Number of times we should run the expiration timer on this connection + * before removing it. */ + int expirationCount; +}; -/* - * Allocate a hash table for fs handles. Returns 0 on success, - * -1 on failure. - */ -int allocFsTable(void) { - assert(NULL == fsTable); - fsTable = calloc(1, sizeof(struct hsearch_data)); - if (0 == hcreate_r(MAX_ELEMENTS, fsTable)) { - ERROR("Unable to initialize connection table"); - return -1; - } - return 0; -} +RB_GENERATE(hdfsConnTree, hdfsConn, entry, hdfsConnCompare); -/* - * Find a fs handle for the given key. Returns a fs handle, - * or NULL if there is no fs for the given key. - */ -static hdfsFS findFs(char *key) { - ENTRY entry; - ENTRY *entryP = NULL; - entry.key = key; - if (0 == hsearch_r(entry, FIND, &entryP, fsTable)) { - return NULL; - } - assert(NULL != entryP->data); - return (hdfsFS)entryP->data; -} +/** Current cached libhdfs connections */ +static struct hdfsConnTree gConnTree; -/* - * Insert the given fs handle into the table. - * Returns 0 on success, -1 on failure. - */ -static int insertFs(char *key, hdfsFS fs) { - ENTRY entry; - ENTRY *entryP = NULL; - assert(NULL != fs); - entry.key = strdup(key); - if (entry.key == NULL) { - return -1; - } - entry.data = (void*)fs; - if (0 == hsearch_r(entry, ENTER, &entryP, fsTable)) { - return -1; - } - return 0; -} +/** The URI used to make our connections. Dynamically allocated. */ +static char *gUri; + +/** The port used to make our connections, or 0. */ +static int gPort; + +/** Lock which protects gConnTree and gConnectTimer->active */ +static pthread_mutex_t gConnMutex; + +/** Type of authentication configured */ +static enum authConf gHdfsAuthConf; + +/** FUSE connection timer expiration period */ +static int32_t gTimerPeriod; + +/** FUSE connection expiry period */ +static int32_t gExpiryPeriod; + +/** FUSE timer expiration thread */ +static pthread_t gTimerThread; /** * Find out what type of authentication the system administrator @@ -95,19 +112,251 @@ static int insertFs(char *key, hdfsFS fs) { */ static enum authConf discoverAuthConf(void) { - int ret; - char *val = NULL; - enum authConf authConf; + int ret; + char *val = NULL; + enum authConf authConf; - ret = hdfsConfGet(HADOOP_SECURITY_AUTHENTICATION, &val); - if (ret) - authConf = AUTH_CONF_UNKNOWN; - else if (!strcmp(val, "kerberos")) - authConf = AUTH_CONF_KERBEROS; - else - authConf = AUTH_CONF_OTHER; - free(val); - return authConf; + ret = hdfsConfGetStr(HADOOP_SECURITY_AUTHENTICATION, &val); + if (ret) + authConf = AUTH_CONF_UNKNOWN; + else if (!val) + authConf = AUTH_CONF_OTHER; + else if (!strcmp(val, "kerberos")) + authConf = AUTH_CONF_KERBEROS; + else + authConf = AUTH_CONF_OTHER; + free(val); + return authConf; +} + +int fuseConnectInit(const char *nnUri, int port) +{ + const char *timerPeriod; + int ret; + + gTimerPeriod = FUSE_CONN_DEFAULT_TIMER_PERIOD; + ret = hdfsConfGetInt(HADOOP_FUSE_CONNECTION_TIMEOUT, &gTimerPeriod); + if (ret) { + fprintf(stderr, "Unable to determine the configured value for %s.", + HADOOP_FUSE_TIMER_PERIOD); + return -EINVAL; + } + if (gTimerPeriod < 1) { + fprintf(stderr, "Invalid value %d given for %s.\n", + gTimerPeriod, HADOOP_FUSE_TIMER_PERIOD); + return -EINVAL; + } + gExpiryPeriod = FUSE_CONN_DEFAULT_EXPIRY_PERIOD; + ret = hdfsConfGetInt(HADOOP_FUSE_CONNECTION_TIMEOUT, &gExpiryPeriod); + if (ret) { + fprintf(stderr, "Unable to determine the configured value for %s.", + HADOOP_FUSE_CONNECTION_TIMEOUT); + return -EINVAL; + } + if (gExpiryPeriod < 1) { + fprintf(stderr, "Invalid value %d given for %s.\n", + gExpiryPeriod, HADOOP_FUSE_CONNECTION_TIMEOUT); + return -EINVAL; + } + gHdfsAuthConf = discoverAuthConf(); + if (gHdfsAuthConf == AUTH_CONF_UNKNOWN) { + fprintf(stderr, "Unable to determine the configured value for %s.", + HADOOP_SECURITY_AUTHENTICATION); + return -EINVAL; + } + gPort = port; + gUri = strdup(nnUri); + if (!gUri) { + fprintf(stderr, "fuseConnectInit: OOM allocting nnUri\n"); + return -ENOMEM; + } + ret = pthread_mutex_init(&gConnMutex, NULL); + if (ret) { + free(gUri); + fprintf(stderr, "fuseConnectInit: pthread_mutex_init failed with error %d\n", + ret); + return -ret; + } + RB_INIT(&gConnTree); + ret = pthread_create(&gTimerThread, NULL, hdfsConnExpiryThread, NULL); + if (ret) { + free(gUri); + pthread_mutex_destroy(&gConnMutex); + fprintf(stderr, "fuseConnectInit: pthread_create failed with error %d\n", + ret); + return -ret; + } + fprintf(stderr, "fuseConnectInit: initialized with timer period %d, " + "expiry period %d\n", gTimerPeriod, gExpiryPeriod); + return 0; +} + +/** + * Compare two libhdfs connections by username + * + * @param a The first libhdfs connection + * @param b The second libhdfs connection + * + * @return -1, 0, or 1 depending on a < b, a ==b, a > b + */ +static int hdfsConnCompare(const struct hdfsConn *a, const struct hdfsConn *b) +{ + return strcmp(a->usrname, b->usrname); +} + +/** + * Find a libhdfs connection by username + * + * @param usrname The username to look up + * + * @return The connection, or NULL if none could be found + */ +static struct hdfsConn* hdfsConnFind(const char *usrname) +{ + struct hdfsConn exemplar; + + memset(&exemplar, 0, sizeof(exemplar)); + exemplar.usrname = (char*)usrname; + return RB_FIND(hdfsConnTree, &gConnTree, &exemplar); +} + +/** + * Free the resource associated with a libhdfs connection. + * + * You must remove the connection from the tree before calling this function. + * + * @param conn The libhdfs connection + */ +static void hdfsConnFree(struct hdfsConn *conn) +{ + int ret; + + ret = hdfsDisconnect(conn->fs); + if (ret) { + fprintf(stderr, "hdfsConnFree(username=%s): " + "hdfsDisconnect failed with error %d\n", + (conn->usrname ? conn->usrname : "(null)"), ret); + } + free(conn->usrname); + free(conn->kpath); + free(conn); +} + +/** + * Convert a time_t to a string. + * + * @param sec time in seconds since the epoch + * @param buf (out param) output buffer + * @param bufLen length of output buffer + * + * @return 0 on success; ENAMETOOLONG if the provided buffer was + * too short + */ +static int timeToStr(time_t sec, char *buf, size_t bufLen) +{ + struct tm tm, *out; + size_t l; + + if (bufLen < TIME_STR_LEN) { + return -ENAMETOOLONG; + } + out = localtime_r(&sec, &tm); + asctime_r(out, buf); + // strip trailing newline + l = strlen(buf); + if (l != 0) + buf[l - 1] = '\0'; + return 0; +} + +/** + * Check an HDFS connection's Kerberos path. + * + * If the mtime of the Kerberos ticket cache file has changed since we first + * opened the connection, mark the connection as condemned and remove it from + * the hdfs connection tree. + * + * @param conn The HDFS connection + */ +static int hdfsConnCheckKpath(const struct hdfsConn *conn) +{ + int ret; + struct stat st; + char prevTimeBuf[TIME_STR_LEN], newTimeBuf[TIME_STR_LEN]; + + if (stat(conn->kpath, &st) < 0) { + ret = errno; + if (ret == ENOENT) { + fprintf(stderr, "hdfsConnCheckKpath(conn.usrname=%s): the kerberos " + "ticket cache file '%s' has disappeared. Condemning the " + "connection.\n", conn->usrname, conn->kpath); + } else { + fprintf(stderr, "hdfsConnCheckKpath(conn.usrname=%s): stat(%s) " + "failed with error code %d. Pessimistically condemning the " + "connection.\n", conn->usrname, conn->kpath, ret); + } + return -ret; + } + if ((st.st_mtim.tv_sec != conn->kPathMtime) || + (st.st_mtim.tv_nsec != conn->kPathMtimeNs)) { + timeToStr(conn->kPathMtime, prevTimeBuf, sizeof(prevTimeBuf)); + timeToStr(st.st_mtim.tv_sec, newTimeBuf, sizeof(newTimeBuf)); + fprintf(stderr, "hdfsConnCheckKpath(conn.usrname=%s): mtime on '%s' " + "has changed from '%s' to '%s'. Condemning the connection " + "because our cached Kerberos credentials have probably " + "changed.\n", conn->usrname, conn->kpath, prevTimeBuf, newTimeBuf); + return -EINTERNAL; + } + return 0; +} + +/** + * Cache expiration logic. + * + * This function is called periodically by the cache expiration thread. For + * each FUSE connection not currently in use (refcnt == 0) it will decrement the + * expirationCount for that connection. Once the expirationCount reaches 0 for + * a connection, it can be garbage collected. + * + * We also check to see if the Kerberos credentials have changed. If so, the + * connecton is immediately condemned, even if it is currently in use. + */ +static void hdfsConnExpiry(void) +{ + struct hdfsConn *conn, *tmpConn; + + pthread_mutex_lock(&gConnMutex); + RB_FOREACH_SAFE(conn, hdfsConnTree, &gConnTree, tmpConn) { + if (conn->kpath) { + if (hdfsConnCheckKpath(conn)) { + conn->condemned = 1; + RB_REMOVE(hdfsConnTree, &gConnTree, conn); + if (conn->refcnt == 0) { + /* If the connection is not in use by any threads, delete it + * immediately. If it is still in use by some threads, the last + * thread using it will clean it up later inside hdfsConnRelease. */ + hdfsConnFree(conn); + continue; + } + } + } + if (conn->refcnt == 0) { + /* If the connection is not currently in use by a thread, check to see if + * it ought to be removed because it's too old. */ + conn->expirationCount--; + if (conn->expirationCount <= 0) { + if (conn->condemned) { + fprintf(stderr, "hdfsConnExpiry: LOGIC ERROR: condemned connection " + "as %s is still in the tree!\n", conn->usrname); + } + fprintf(stderr, "hdfsConnExpiry: freeing and removing connection as " + "%s because it's now too old.\n", conn->usrname); + RB_REMOVE(hdfsConnTree, &gConnTree, conn); + hdfsConnFree(conn); + } + } + } + pthread_mutex_unlock(&gConnMutex); } /** @@ -129,9 +378,9 @@ static enum authConf discoverAuthConf(void) * @param path (out param) the path to the ticket cache file * @param pathLen length of the path buffer */ -static void findKerbTicketCachePath(char *path, size_t pathLen) +static void findKerbTicketCachePath(struct fuse_context *ctx, + char *path, size_t pathLen) { - struct fuse_context *ctx = fuse_get_context(); FILE *fp = NULL; static const char * const KRB5CCNAME = "\0KRB5CCNAME="; int c = '\0', pathIdx = 0, keyIdx = 0; @@ -168,72 +417,213 @@ done: } } -/* - * Connect to the NN as the current user/group. - * Returns a fs handle on success, or NULL on failure. +/** + * Create a new libhdfs connection. + * + * @param usrname Username to use for the new connection + * @param ctx FUSE context to use for the new connection + * @param out (out param) the new libhdfs connection + * + * @return 0 on success; error code otherwise */ -hdfsFS doConnectAsUser(const char *nn_uri, int nn_port) { - struct hdfsBuilder *bld; - uid_t uid = fuse_get_context()->uid; - char *user = getUsername(uid); - char kpath[PATH_MAX]; +static int fuseNewConnect(const char *usrname, struct fuse_context *ctx, + struct hdfsConn **out) +{ + struct hdfsBuilder *bld = NULL; + char kpath[PATH_MAX] = { 0 }; + struct hdfsConn *conn = NULL; int ret; - hdfsFS fs = NULL; - if (NULL == user) { - goto done; + struct stat st; + + conn = calloc(1, sizeof(struct hdfsConn)); + if (!conn) { + fprintf(stderr, "fuseNewConnect: OOM allocating struct hdfsConn\n"); + ret = -ENOMEM; + goto error; } - - ret = pthread_mutex_lock(&tableMutex); - assert(0 == ret); - - fs = findFs(user); - if (NULL == fs) { - if (hdfsAuthConf == AUTH_CONF_UNKNOWN) { - hdfsAuthConf = discoverAuthConf(); - if (hdfsAuthConf == AUTH_CONF_UNKNOWN) { - ERROR("Unable to determine the configured value for %s.", - HADOOP_SECURITY_AUTHENTICATION); - goto done; - } + bld = hdfsNewBuilder(); + if (!bld) { + fprintf(stderr, "Unable to create hdfs builder\n"); + ret = -ENOMEM; + goto error; + } + /* We always want to get a new FileSystem instance here-- that's why we call + * hdfsBuilderSetForceNewInstance. Otherwise the 'cache condemnation' logic + * in hdfsConnExpiry will not work correctly, since FileSystem might re-use the + * existing cached connection which we wanted to get rid of. + */ + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetNameNode(bld, gUri); + if (gPort) { + hdfsBuilderSetNameNodePort(bld, gPort); + } + hdfsBuilderSetUserName(bld, usrname); + if (gHdfsAuthConf == AUTH_CONF_KERBEROS) { + findKerbTicketCachePath(ctx, kpath, sizeof(kpath)); + if (stat(kpath, &st) < 0) { + fprintf(stderr, "fuseNewConnect: failed to find Kerberos ticket cache " + "file '%s'. Did you remember to kinit for UID %d?\n", + kpath, ctx->uid); + ret = -EACCES; + goto error; } - bld = hdfsNewBuilder(); - if (!bld) { - ERROR("Unable to create hdfs builder"); - goto done; - } - hdfsBuilderSetForceNewInstance(bld); - hdfsBuilderSetNameNode(bld, nn_uri); - if (nn_port) { - hdfsBuilderSetNameNodePort(bld, nn_port); - } - hdfsBuilderSetUserName(bld, user); - if (hdfsAuthConf == AUTH_CONF_KERBEROS) { - findKerbTicketCachePath(kpath, sizeof(kpath)); - hdfsBuilderSetKerbTicketCachePath(bld, kpath); - } - fs = hdfsBuilderConnect(bld); - if (NULL == fs) { - int err = errno; - ERROR("Unable to create fs for user %s: error code %d", user, err); - goto done; - } - if (-1 == insertFs(user, fs)) { - ERROR("Unable to cache fs for user %s", user); + conn->kPathMtime = st.st_mtim.tv_sec; + conn->kPathMtimeNs = st.st_mtim.tv_nsec; + hdfsBuilderSetKerbTicketCachePath(bld, kpath); + conn->kpath = strdup(kpath); + if (!conn->kpath) { + fprintf(stderr, "fuseNewConnect: OOM allocating kpath\n"); + ret = -ENOMEM; + goto error; } } + conn->usrname = strdup(usrname); + if (!conn->usrname) { + fprintf(stderr, "fuseNewConnect: OOM allocating usrname\n"); + ret = -ENOMEM; + goto error; + } + conn->fs = hdfsBuilderConnect(bld); + bld = NULL; + if (!conn->fs) { + ret = errno; + fprintf(stderr, "fuseNewConnect(usrname=%s): Unable to create fs: " + "error code %d\n", usrname, ret); + goto error; + } + RB_INSERT(hdfsConnTree, &gConnTree, conn); + *out = conn; + return 0; -done: - ret = pthread_mutex_unlock(&tableMutex); - assert(0 == ret); - free(user); - return fs; +error: + if (bld) { + hdfsFreeBuilder(bld); + } + if (conn) { + free(conn->kpath); + free(conn->usrname); + free(conn); + } + return ret; } -/* - * We currently cache a fs handle per-user in this module rather - * than use the FileSystem cache in the java client. Therefore - * we do not disconnect the fs handle here. - */ -int doDisconnect(hdfsFS fs) { +int fuseConnect(const char *usrname, struct fuse_context *ctx, + struct hdfsConn **out) +{ + int ret; + struct hdfsConn* conn; + + pthread_mutex_lock(&gConnMutex); + conn = hdfsConnFind(usrname); + if (!conn) { + ret = fuseNewConnect(usrname, ctx, &conn); + if (ret) { + pthread_mutex_unlock(&gConnMutex); + fprintf(stderr, "fuseConnect(usrname=%s): fuseNewConnect failed with " + "error code %d\n", usrname, ret); + return ret; + } + } + conn->refcnt++; + conn->expirationCount = (gExpiryPeriod + gTimerPeriod - 1) / gTimerPeriod; + if (conn->expirationCount < 2) + conn->expirationCount = 2; + pthread_mutex_unlock(&gConnMutex); + *out = conn; return 0; } + +int fuseConnectAsThreadUid(struct hdfsConn **conn) +{ + struct fuse_context *ctx; + char *usrname; + int ret; + + ctx = fuse_get_context(); + usrname = getUsername(ctx->uid); + ret = fuseConnect(usrname, ctx, conn); + free(usrname); + return ret; +} + +int fuseConnectTest(void) +{ + int ret; + struct hdfsConn *conn; + + if (gHdfsAuthConf == AUTH_CONF_KERBEROS) { + // TODO: call some method which can tell us whether the FS exists. In order + // to implement this, we have to add a method to FileSystem in order to do + // this without valid Kerberos authentication. See HDFS-3674 for details. + return 0; + } + ret = fuseNewConnect("root", NULL, &conn); + if (ret) { + fprintf(stderr, "fuseConnectTest failed with error code %d\n", ret); + return ret; + } + hdfsConnRelease(conn); + return 0; +} + +struct hdfs_internal* hdfsConnGetFs(struct hdfsConn *conn) +{ + return conn->fs; +} + +void hdfsConnRelease(struct hdfsConn *conn) +{ + pthread_mutex_lock(&gConnMutex); + conn->refcnt--; + if ((conn->refcnt == 0) && (conn->condemned)) { + fprintf(stderr, "hdfsConnRelease(usrname=%s): freeing condemend FS!\n", + conn->usrname); + /* Notice that we're not removing the connection from gConnTree here. + * If the connection is condemned, it must have already been removed from + * the tree, so that no other threads start using it. + */ + hdfsConnFree(conn); + } + pthread_mutex_unlock(&gConnMutex); +} + +/** + * Get the monotonic time. + * + * Unlike the wall-clock time, monotonic time only ever goes forward. If the + * user adjusts the time, the monotonic time will not be affected. + * + * @return The monotonic time + */ +static time_t getMonotonicTime(void) +{ + int res; + struct timespec ts; + + res = clock_gettime(CLOCK_MONOTONIC, &ts); + if (res) + abort(); + return ts.tv_sec; +} + +/** + * FUSE connection expiration thread + * + */ +static void* hdfsConnExpiryThread(void *v) +{ + time_t nextTime, curTime; + int waitTime; + + nextTime = getMonotonicTime() + gTimerPeriod; + while (1) { + curTime = getMonotonicTime(); + if (curTime >= nextTime) { + hdfsConnExpiry(); + nextTime = curTime + gTimerPeriod; + } + waitTime = (nextTime - curTime) * 1000; + poll(NULL, 0, waitTime); + } + return NULL; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.h index 4bddeeae9f0..35645c66b6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.h @@ -19,10 +19,72 @@ #ifndef __FUSE_CONNECT_H__ #define __FUSE_CONNECT_H__ -#include "fuse_dfs.h" +struct fuse_context; +struct hdfsConn; +struct hdfs_internal; -hdfsFS doConnectAsUser(const char *nn_uri, int nn_port); -int doDisconnect(hdfsFS fs); -int allocFsTable(void); +/** + * Initialize the fuse connection subsystem. + * + * This must be called before any of the other functions in this module. + * + * @param nnUri The NameNode URI + * @param port The NameNode port + * + * @return 0 on success; error code otherwise + */ +int fuseConnectInit(const char *nnUri, int port); + +/** + * Get a libhdfs connection. + * + * If there is an existing connection, it will be reused. If not, a new one + * will be created. + * + * You must call hdfsConnRelease on the connection you get back! + * + * @param usrname The username to use + * @param ctx The FUSE context to use (contains UID, PID of requestor) + * @param conn (out param) The HDFS connection + * + * @return 0 on success; error code otherwise + */ +int fuseConnect(const char *usrname, struct fuse_context *ctx, + struct hdfsConn **out); + +/** + * Get a libhdfs connection. + * + * The same as fuseConnect, except the username will be determined from the FUSE + * thread context. + * + * @param conn (out param) The HDFS connection + * + * @return 0 on success; error code otherwise + */ +int fuseConnectAsThreadUid(struct hdfsConn **conn); + +/** + * Test whether we can connect to the HDFS cluster + * + * @return 0 on success; error code otherwise + */ +int fuseConnectTest(void); + +/** + * Get the hdfsFS associated with an hdfsConn. + * + * @param conn The hdfsConn + * + * @return the hdfsFS + */ +struct hdfs_internal* hdfsConnGetFs(struct hdfsConn *conn); + +/** + * Release an hdfsConn when we're done with it. + * + * @param conn The hdfsConn + */ +void hdfsConnRelease(struct hdfsConn *conn); #endif diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_context_handle.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_context_handle.h index ae07735d88e..f2b48be29c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_context_handle.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_context_handle.h @@ -31,8 +31,6 @@ // typedef struct dfs_context_struct { int debug; - char *nn_uri; - int nn_port; int read_only; int usetrash; int direct_io; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c index e218c81c9ec..1a8ede6348e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c @@ -65,8 +65,19 @@ static struct fuse_operations dfs_oper = { .truncate = dfs_truncate, }; +static void print_env_vars(void) +{ + const char *cp = getenv("CLASSPATH"); + const char *ld = getenv("LD_LIBRARY_PATH"); + + fprintf(stderr, "LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld); + fprintf(stderr, "CLASSPATH=%s",cp == NULL ? "NULL" : cp); +} + int main(int argc, char *argv[]) { + int ret; + umask(0); extern const char *program; @@ -106,24 +117,22 @@ int main(int argc, char *argv[]) exit(0); } - // Check connection as root + ret = fuseConnectInit(options.nn_uri, options.nn_port); + if (ret) { + ERROR("FATAL: dfs_init: fuseConnInit failed with error %d!", ret); + print_env_vars(); + exit(EXIT_FAILURE); + } if (options.initchecks == 1) { - hdfsFS tempFS = hdfsConnectAsUser(options.nn_uri, options.nn_port, "root"); - if (NULL == tempFS) { - const char *cp = getenv("CLASSPATH"); - const char *ld = getenv("LD_LIBRARY_PATH"); - ERROR("FATAL: misconfiguration - cannot connect to HDFS"); - ERROR("LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld); - ERROR("CLASSPATH=%s",cp == NULL ? "NULL" : cp); - exit(1); - } - if (doDisconnect(tempFS)) { - ERROR("FATAL: unable to disconnect from test filesystem."); - exit(1); + ret = fuseConnectTest(); + if (ret) { + ERROR("FATAL: dfs_init: fuseConnTest failed with error %d!", ret); + print_env_vars(); + exit(EXIT_FAILURE); } } - int ret = fuse_main(args.argc, args.argv, &dfs_oper, NULL); + ret = fuse_main(args.argc, args.argv, &dfs_oper, NULL); fuse_opt_free_args(&args); return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_file_handle.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_file_handle.h index 70cd8983a81..7f9346c1e9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_file_handle.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_file_handle.h @@ -22,6 +22,8 @@ #include #include +struct hdfsConn; + /** * * dfs_fh_struct is passed around for open files. Fuse provides a hook (the context) @@ -34,10 +36,10 @@ */ typedef struct dfs_fh_struct { hdfsFile hdfsFH; + struct hdfsConn *conn; char *buf; tSize bufferSize; //what is the size of the buffer we have off_t buffersStartOffset; //where the buffer starts in the file - hdfsFS fs; // for reads/writes need to access as the real user pthread_mutex_t mutex; } dfs_fh; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chmod.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chmod.c index 2c1e96b2c1b..8c25f53b6a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chmod.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chmod.c @@ -23,6 +23,8 @@ int dfs_chmod(const char *path, mode_t mode) { + struct hdfsConn *conn = NULL; + hdfsFS fs; TRACE1("chmod", path) int ret = 0; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; @@ -31,22 +33,24 @@ int dfs_chmod(const char *path, mode_t mode) assert(dfs); assert('/' == *path); - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect to HDFS"); + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); ret = -EIO; goto cleanup; } + fs = hdfsConnGetFs(conn); - if (hdfsChmod(userFS, path, (short)mode)) { + if (hdfsChmod(fs, path, (short)mode)) { ERROR("Could not chmod %s to %d", path, (int)mode); ret = (errno > 0) ? -errno : -EIO; goto cleanup; } cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; + if (conn) { + hdfsConnRelease(conn); } return ret; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chown.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chown.c index a14ca7137e6..2a6b61c027e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chown.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chown.c @@ -25,12 +25,12 @@ int dfs_chown(const char *path, uid_t uid, gid_t gid) { - TRACE1("chown", path) - + struct hdfsConn *conn = NULL; int ret = 0; char *user = NULL; char *group = NULL; - hdfsFS userFS = NULL; + + TRACE1("chown", path) // retrieve dfs specific data dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; @@ -61,14 +61,15 @@ int dfs_chown(const char *path, uid_t uid, gid_t gid) } } - userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect to HDFS"); + ret = fuseConnect(user, fuse_get_context(), &conn); + if (ret) { + fprintf(stderr, "fuseConnect: failed to open a libhdfs connection! " + "error %d.\n", ret); ret = -EIO; goto cleanup; } - if (hdfsChown(userFS, path, user, group)) { + if (hdfsChown(hdfsConnGetFs(conn), path, user, group)) { ret = errno; ERROR("Could not chown %s to %d:%d: error %d", path, (int)uid, gid, ret); ret = (ret > 0) ? -ret : -EIO; @@ -76,16 +77,11 @@ int dfs_chown(const char *path, uid_t uid, gid_t gid) } cleanup: - if (userFS && doDisconnect(userFS)) { - ret = -EIO; - } - if (user) { - free(user); - } - if (group) { - free(group); + if (conn) { + hdfsConnRelease(conn); } + free(user); + free(group); return ret; - } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_flush.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_flush.c index 6d4f05c4ec6..adb065b229a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_flush.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_flush.c @@ -16,6 +16,7 @@ * limitations under the License. */ +#include "fuse_connect.h" #include "fuse_dfs.h" #include "fuse_impls.h" #include "fuse_file_handle.h" @@ -43,9 +44,7 @@ int dfs_flush(const char *path, struct fuse_file_info *fi) { assert(fh); hdfsFile file_handle = (hdfsFile)fh->hdfsFH; assert(file_handle); - - assert(fh->fs); - if (hdfsFlush(fh->fs, file_handle) != 0) { + if (hdfsFlush(hdfsConnGetFs(fh->conn), file_handle) != 0) { ERROR("Could not flush %lx for %s\n",(long)file_handle, path); return -EIO; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_getattr.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_getattr.c index 56f634e71bb..2e435185ec1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_getattr.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_getattr.c @@ -23,22 +23,27 @@ int dfs_getattr(const char *path, struct stat *st) { + struct hdfsConn *conn = NULL; + hdfsFS fs; + int ret; + hdfsFileInfo *info; + TRACE1("getattr", path) - dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; - assert(dfs); assert(path); assert(st); - hdfsFS fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (NULL == fs) { - ERROR("Could not connect to %s:%d", dfs->nn_uri, dfs->nn_port); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } - - int ret = 0; - hdfsFileInfo *info = hdfsGetPathInfo(fs,path); + fs = hdfsConnGetFs(conn); + + info = hdfsGetPathInfo(fs,path); if (NULL == info) { ret = -ENOENT; goto cleanup; @@ -63,9 +68,8 @@ int dfs_getattr(const char *path, struct stat *st) hdfsFreeFileInfo(info,1); cleanup: - if (doDisconnect(fs)) { - ERROR("Could not disconnect from filesystem"); - ret = -EIO; + if (conn) { + hdfsConnRelease(conn); } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_mkdir.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_mkdir.c index d0624afa1c2..3aef108109b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_mkdir.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_mkdir.c @@ -23,9 +23,12 @@ int dfs_mkdir(const char *path, mode_t mode) { - TRACE1("mkdir", path) - + struct hdfsConn *conn = NULL; + hdfsFS fs; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int ret; + + TRACE1("mkdir", path) assert(path); assert(dfs); @@ -41,29 +44,32 @@ int dfs_mkdir(const char *path, mode_t mode) return -EACCES; } - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } + fs = hdfsConnGetFs(conn); // In theory the create and chmod should be atomic. - int ret = 0; - if (hdfsCreateDirectory(userFS, path)) { + if (hdfsCreateDirectory(fs, path)) { ERROR("HDFS could not create directory %s", path); ret = (errno > 0) ? -errno : -EIO; goto cleanup; } - if (hdfsChmod(userFS, path, (short)mode)) { + if (hdfsChmod(fs, path, (short)mode)) { ERROR("Could not chmod %s to %d", path, (int)mode); ret = (errno > 0) ? -errno : -EIO; } + ret = 0; cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; + if (conn) { + hdfsConnRelease(conn); } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_open.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_open.c index 071590aa371..ecd772f63f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_open.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_open.c @@ -21,38 +21,45 @@ #include "fuse_connect.h" #include "fuse_file_handle.h" +#include +#include + int dfs_open(const char *path, struct fuse_file_info *fi) { - TRACE1("open", path) - + hdfsFS fs = NULL; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + dfs_fh *fh = NULL; + int mutexInit = 0, ret; + + TRACE1("open", path) // check params and the context var assert(path); assert('/' == *path); assert(dfs); - int ret = 0; - // 0x8000 is always passed in and hadoop doesn't like it, so killing it here // bugbug figure out what this flag is and report problem to Hadoop JIRA int flags = (fi->flags & 0x7FFF); // retrieve dfs specific data - dfs_fh *fh = (dfs_fh*)calloc(1, sizeof (dfs_fh)); - if (fh == NULL) { + fh = (dfs_fh*)calloc(1, sizeof (dfs_fh)); + if (!fh) { ERROR("Malloc of new file handle failed"); - return -EIO; + ret = -EIO; + goto error; } - - fh->fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (fh->fs == NULL) { - ERROR("Could not connect to dfs"); - return -EIO; + ret = fuseConnectAsThreadUid(&fh->conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto error; } + fs = hdfsConnGetFs(fh->conn); if (flags & O_RDWR) { - hdfsFileInfo *info = hdfsGetPathInfo(fh->fs,path); + hdfsFileInfo *info = hdfsGetPathInfo(fs, path); if (info == NULL) { // File does not exist (maybe?); interpret it as a O_WRONLY // If the actual error was something else, we'll get it again when @@ -66,15 +73,23 @@ int dfs_open(const char *path, struct fuse_file_info *fi) } } - if ((fh->hdfsFH = hdfsOpenFile(fh->fs, path, flags, 0, 0, 0)) == NULL) { + if ((fh->hdfsFH = hdfsOpenFile(fs, path, flags, 0, 0, 0)) == NULL) { ERROR("Could not open file %s (errno=%d)", path, errno); if (errno == 0 || errno == EINTERNAL) { - return -EIO; + ret = -EIO; + goto error; } - return -errno; + ret = -errno; + goto error; } - pthread_mutex_init(&fh->mutex, NULL); + ret = pthread_mutex_init(&fh->mutex, NULL); + if (ret) { + fprintf(stderr, "dfs_open: error initializing mutex: error %d\n", ret); + ret = -EIO; + goto error; + } + mutexInit = 1; if (fi->flags & O_WRONLY || fi->flags & O_CREAT) { fh->buf = NULL; @@ -84,11 +99,27 @@ int dfs_open(const char *path, struct fuse_file_info *fi) if (NULL == fh->buf) { ERROR("Could not allocate memory for a read for file %s\n", path); ret = -EIO; + goto error; } fh->buffersStartOffset = 0; fh->bufferSize = 0; } fi->fh = (uint64_t)fh; + return 0; +error: + if (fh) { + if (mutexInit) { + pthread_mutex_destroy(&fh->mutex); + } + free(fh->buf); + if (fh->hdfsFH) { + hdfsCloseFile(fs, fh->hdfsFH); + } + if (fh->conn) { + hdfsConnRelease(fh->conn); + } + free(fh); + } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_read.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_read.c index 52092616c06..feade454035 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_read.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_read.c @@ -16,9 +16,10 @@ * limitations under the License. */ +#include "fuse_connect.h" #include "fuse_dfs.h" -#include "fuse_impls.h" #include "fuse_file_handle.h" +#include "fuse_impls.h" static size_t min(const size_t x, const size_t y) { return x < y ? x : y; @@ -48,9 +49,9 @@ int dfs_read(const char *path, char *buf, size_t size, off_t offset, assert(fi); dfs_fh *fh = (dfs_fh*)fi->fh; + hdfsFS fs = hdfsConnGetFs(fh->conn); assert(fh != NULL); - assert(fh->fs != NULL); assert(fh->hdfsFH != NULL); // special case this as simplifies the rest of the logic to know the caller wanted > 0 bytes @@ -61,7 +62,7 @@ int dfs_read(const char *path, char *buf, size_t size, off_t offset, if ( size >= dfs->rdbuffer_size) { int num_read; size_t total_read = 0; - while (size - total_read > 0 && (num_read = hdfsPread(fh->fs, fh->hdfsFH, offset + total_read, buf + total_read, size - total_read)) > 0) { + while (size - total_read > 0 && (num_read = hdfsPread(fs, fh->hdfsFH, offset + total_read, buf + total_read, size - total_read)) > 0) { total_read += num_read; } // if there was an error before satisfying the current read, this logic declares it an error @@ -98,7 +99,7 @@ int dfs_read(const char *path, char *buf, size_t size, off_t offset, size_t total_read = 0; while (dfs->rdbuffer_size - total_read > 0 && - (num_read = hdfsPread(fh->fs, fh->hdfsFH, offset + total_read, fh->buf + total_read, dfs->rdbuffer_size - total_read)) > 0) { + (num_read = hdfsPread(fs, fh->hdfsFH, offset + total_read, fh->buf + total_read, dfs->rdbuffer_size - total_read)) > 0) { total_read += num_read; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_readdir.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_readdir.c index f6fe48b6ad9..326f5730509 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_readdir.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_readdir.c @@ -24,25 +24,31 @@ int dfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi) { - TRACE1("readdir", path) + int ret; + struct hdfsConn *conn = NULL; + hdfsFS fs; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + TRACE1("readdir", path) + assert(dfs); assert(path); assert(buf); - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } + fs = hdfsConnGetFs(conn); // Read dirents. Calling a variant that just returns the final path // component (HDFS-975) would save us from parsing it out below. int numEntries = 0; - hdfsFileInfo *info = hdfsListDirectory(userFS, path, &numEntries); + hdfsFileInfo *info = hdfsListDirectory(fs, path, &numEntries); - int ret = 0; // NULL means either the directory doesn't exist or maybe IO error. if (NULL == info) { ret = (errno > 0) ? -errno : -ENOENT; @@ -106,11 +112,11 @@ int dfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, } // free the info pointers hdfsFreeFileInfo(info,numEntries); + ret = 0; cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; - ERROR("Failed to disconnect %d", errno); + if (conn) { + hdfsConnRelease(conn); } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_release.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_release.c index e15dd572d63..0316de61410 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_release.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_release.c @@ -52,15 +52,13 @@ int dfs_release (const char *path, struct fuse_file_info *fi) { assert(fh); hdfsFile file_handle = (hdfsFile)fh->hdfsFH; if (NULL != file_handle) { - if (hdfsCloseFile(fh->fs, file_handle) != 0) { + if (hdfsCloseFile(hdfsConnGetFs(fh->conn), file_handle) != 0) { ERROR("Could not close handle %ld for %s\n",(long)file_handle, path); ret = -EIO; } } free(fh->buf); - if (doDisconnect(fh->fs)) { - ret = -EIO; - } + hdfsConnRelease(fh->conn); pthread_mutex_destroy(&fh->mutex); free(fh); fi->fh = 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rename.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rename.c index bbb0462b038..415539f0abc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rename.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rename.c @@ -23,10 +23,12 @@ int dfs_rename(const char *from, const char *to) { - TRACE1("rename", from) - - // retrieve dfs specific data + struct hdfsConn *conn = NULL; + hdfsFS fs; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int ret; + + TRACE1("rename", from) // check params and the context var assert(from); @@ -46,23 +48,24 @@ int dfs_rename(const char *from, const char *to) return -EACCES; } - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } - - int ret = 0; - if (hdfsRename(userFS, from, to)) { + fs = hdfsConnGetFs(conn); + if (hdfsRename(fs, from, to)) { ERROR("Rename %s to %s failed", from, to); ret = (errno > 0) ? -errno : -EIO; goto cleanup; } + ret = 0; cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; + if (conn) { + hdfsConnRelease(conn); } return ret; - } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rmdir.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rmdir.c index 259040fe9fc..f79562a8b50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rmdir.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rmdir.c @@ -25,9 +25,14 @@ extern const char *const TrashPrefixDir; int dfs_rmdir(const char *path) { - TRACE1("rmdir", path) - + struct hdfsConn *conn = NULL; + hdfsFS fs; + int ret; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int numEntries = 0; + hdfsFileInfo *info = NULL; + + TRACE1("rmdir", path) assert(path); assert(dfs); @@ -35,42 +40,43 @@ int dfs_rmdir(const char *path) if (is_protected(path)) { ERROR("Trying to delete protected directory %s", path); - return -EACCES; + ret = -EACCES; + goto cleanup; } if (dfs->read_only) { ERROR("HDFS configured read-only, cannot delete directory %s", path); - return -EACCES; + ret = -EACCES; + goto cleanup; } - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } - - int ret = 0; - int numEntries = 0; - hdfsFileInfo *info = hdfsListDirectory(userFS,path,&numEntries); - - if (info) { - hdfsFreeFileInfo(info, numEntries); - } - + fs = hdfsConnGetFs(conn); + info = hdfsListDirectory(fs, path, &numEntries); if (numEntries) { ret = -ENOTEMPTY; goto cleanup; } - if (hdfsDeleteWithTrash(userFS, path, dfs->usetrash)) { + if (hdfsDeleteWithTrash(fs, path, dfs->usetrash)) { ERROR("Error trying to delete directory %s", path); ret = -EIO; goto cleanup; } + ret = 0; cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; + if (info) { + hdfsFreeFileInfo(info, numEntries); + } + if (conn) { + hdfsConnRelease(conn); } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_statfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_statfs.c index c7004a96990..c9306a21270 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_statfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_statfs.c @@ -23,9 +23,12 @@ int dfs_statfs(const char *path, struct statvfs *st) { - TRACE1("statfs",path) - + struct hdfsConn *conn = NULL; + hdfsFS fs; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int ret; + + TRACE1("statfs",path) assert(path); assert(st); @@ -33,19 +36,18 @@ int dfs_statfs(const char *path, struct statvfs *st) memset(st,0,sizeof(struct statvfs)); - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } + fs = hdfsConnGetFs(conn); - const tOffset cap = hdfsGetCapacity(userFS); - const tOffset used = hdfsGetUsed(userFS); - const tOffset bsize = hdfsGetDefaultBlockSize(userFS); - - if (doDisconnect(userFS)) { - return -EIO; - } + const tOffset cap = hdfsGetCapacity(fs); + const tOffset used = hdfsGetUsed(fs); + const tOffset bsize = hdfsGetDefaultBlockSize(fs); st->f_bsize = bsize; st->f_frsize = bsize; @@ -58,6 +60,11 @@ int dfs_statfs(const char *path, struct statvfs *st) st->f_fsid = 1023; st->f_flag = ST_RDONLY | ST_NOSUID; st->f_namemax = 1023; + ret = 0; - return 0; +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_truncate.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_truncate.c index d09b0c8fa63..bf72ca6bec9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_truncate.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_truncate.c @@ -28,10 +28,12 @@ */ int dfs_truncate(const char *path, off_t size) { - TRACE1("truncate", path) - + struct hdfsConn *conn = NULL; + hdfsFS fs; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + TRACE1("truncate", path) + assert(path); assert('/' == *path); assert(dfs); @@ -45,31 +47,33 @@ int dfs_truncate(const char *path, off_t size) return ret; } - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); ret = -EIO; goto cleanup; } + fs = hdfsConnGetFs(conn); int flags = O_WRONLY | O_CREAT; hdfsFile file; - if ((file = (hdfsFile)hdfsOpenFile(userFS, path, flags, 0, 0, 0)) == NULL) { + if ((file = (hdfsFile)hdfsOpenFile(fs, path, flags, 0, 0, 0)) == NULL) { ERROR("Could not connect open file %s", path); ret = -EIO; goto cleanup; } - if (hdfsCloseFile(userFS, file) != 0) { + if (hdfsCloseFile(fs, file) != 0) { ERROR("Could not close file %s", path); ret = -EIO; goto cleanup; } cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; + if (conn) { + hdfsConnRelease(conn); } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_unlink.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_unlink.c index a3d2034dbbd..102c2cd0f3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_unlink.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_unlink.c @@ -20,44 +20,51 @@ #include "fuse_impls.h" #include "fuse_connect.h" #include "fuse_trash.h" -extern const char *const TrashPrefixDir; int dfs_unlink(const char *path) { - TRACE1("unlink", path) - + struct hdfsConn *conn = NULL; + hdfsFS fs; int ret = 0; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + TRACE1("unlink", path) + assert(path); assert(dfs); assert('/' == *path); if (is_protected(path)) { ERROR("Trying to delete protected directory %s", path); - return -EACCES; + ret = -EACCES; + goto cleanup; } if (dfs->read_only) { ERROR("HDFS configured read-only, cannot create directory %s", path); - return -EACCES; + ret = -EACCES; + goto cleanup; } - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } + fs = hdfsConnGetFs(conn); - if (hdfsDeleteWithTrash(userFS, path, dfs->usetrash)) { + if (hdfsDeleteWithTrash(fs, path, dfs->usetrash)) { ERROR("Could not delete file %s", path); ret = (errno > 0) ? -errno : -EIO; goto cleanup; } + ret = 0; cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; + if (conn) { + hdfsConnRelease(conn); } return ret; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_utimens.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_utimens.c index f9144f8451c..dccff92c3fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_utimens.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_utimens.c @@ -22,10 +22,13 @@ int dfs_utimens(const char *path, const struct timespec ts[2]) { - TRACE1("utimens", path) + struct hdfsConn *conn = NULL; + hdfsFS fs; int ret = 0; dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + TRACE1("utimens", path) + assert(path); assert(dfs); assert('/' == *path); @@ -33,14 +36,17 @@ int dfs_utimens(const char *path, const struct timespec ts[2]) time_t aTime = ts[0].tv_sec; time_t mTime = ts[1].tv_sec; - hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port); - if (userFS == NULL) { - ERROR("Could not connect"); - return -EIO; + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; } + fs = hdfsConnGetFs(conn); - if (hdfsUtime(userFS, path, mTime, aTime)) { - hdfsFileInfo *info = hdfsGetPathInfo(userFS, path); + if (hdfsUtime(fs, path, mTime, aTime)) { + hdfsFileInfo *info = hdfsGetPathInfo(fs, path); if (info == NULL) { ret = (errno > 0) ? -errno : -ENOENT; goto cleanup; @@ -54,10 +60,11 @@ int dfs_utimens(const char *path, const struct timespec ts[2]) } goto cleanup; } + ret = 0; cleanup: - if (doDisconnect(userFS)) { - ret = -EIO; + if (conn) { + hdfsConnRelease(conn); } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_write.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_write.c index 8bb0454888d..3090e9e32bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_write.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_write.c @@ -16,6 +16,7 @@ * limitations under the License. */ +#include "fuse_connect.h" #include "fuse_dfs.h" #include "fuse_impls.h" #include "fuse_file_handle.h" @@ -48,15 +49,15 @@ int dfs_write(const char *path, const char *buf, size_t size, pthread_mutex_lock(&fh->mutex); tSize length = 0; - assert(fh->fs); + hdfsFS fs = hdfsConnGetFs(fh->conn); - tOffset cur_offset = hdfsTell(fh->fs, file_handle); + tOffset cur_offset = hdfsTell(fs, file_handle); if (cur_offset != offset) { ERROR("User trying to random access write to a file %d != %d for %s", (int)cur_offset, (int)offset, path); ret = -ENOTSUP; } else { - length = hdfsWrite(fh->fs, file_handle, buf, size); + length = hdfsWrite(fs, file_handle, buf, size); if (length <= 0) { ERROR("Could not write all bytes for %s %d != %d (errno=%d)", path, length, (int)size, errno); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c index 6c1c0d058a3..aeb5f386a6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c @@ -78,8 +78,20 @@ void init_protectedpaths(dfs_context *dfs) { dfs->protectedpaths[j] = NULL; } +static void dfsPrintOptions(FILE *fp, const struct options *o) +{ + fprintf(fp, "[ protected=%s, nn_uri=%s, nn_port=%d, " + "debug=%d, read_only=%d, initchecks=%d, " + "no_permissions=%d, usetrash=%d, entry_timeout=%d, " + "attribute_timeout=%d, rdbuffer_size=%Zd, direct_io=%d ]", + (o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port, + o->debug, o->read_only, o->initchecks, + o->no_permissions, o->usetrash, o->entry_timeout, + o->attribute_timeout, o->rdbuffer_size, o->direct_io); +} -void *dfs_init(void) { +void *dfs_init(void) +{ // // Create a private struct of data we will pass to fuse here and which // will then be accessible on every call. @@ -92,15 +104,15 @@ void *dfs_init(void) { // initialize the context dfs->debug = options.debug; - dfs->nn_uri = options.nn_uri; - dfs->nn_port = options.nn_port; dfs->read_only = options.read_only; dfs->usetrash = options.usetrash; dfs->protectedpaths = NULL; dfs->rdbuffer_size = options.rdbuffer_size; dfs->direct_io = options.direct_io; - INFO("Mounting. nn_uri=%s, nn_port=%d", dfs->nn_uri, dfs->nn_port); + fprintf(stderr, "Mounting with options "); + dfsPrintOptions(stderr, &options); + fprintf(stderr, "\n"); init_protectedpaths(dfs); assert(dfs->protectedpaths != NULL); @@ -109,12 +121,6 @@ void *dfs_init(void) { DEBUG("dfs->rdbuffersize <= 0 = %ld", dfs->rdbuffer_size); dfs->rdbuffer_size = 32768; } - - if (0 != allocFsTable()) { - ERROR("FATAL: could not allocate "); - exit(1); - } - return (void*)dfs; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c index eba2bf1b66e..70468f80c02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c @@ -225,7 +225,7 @@ done: * * @return 0 on success; error code otherwise */ -static int hadoopConfSet(JNIEnv *env, jobject jConfiguration, +static int hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, const char *key, const char *value) { int ret; @@ -283,7 +283,7 @@ static int jStrToCstr(JNIEnv *env, jstring jstr, char **cstr) return 0; } -static int hadoopConfGet(JNIEnv *env, jobject jConfiguration, +static int hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, const char *key, char **val) { int ret; @@ -301,7 +301,7 @@ static int hadoopConfGet(JNIEnv *env, jobject jConfiguration, HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING)), jkey); if (ret) { - snprintf(buf, sizeof(buf), "hadoopConfGet(%s)", key); + snprintf(buf, sizeof(buf), "hadoopConfGetStr(%s)", key); ret = errnoFromException(jExc, env, buf); goto done; } @@ -321,7 +321,7 @@ done: return ret; } -int hdfsConfGet(const char *key, char **val) +int hdfsConfGetStr(const char *key, char **val) { JNIEnv *env; int ret; @@ -339,19 +339,67 @@ int hdfsConfGet(const char *key, char **val) ret = EINTERNAL; goto done; } - ret = hadoopConfGet(env, jConfiguration, key, val); - if (ret) - goto done; - ret = 0; + ret = hadoopConfGetStr(env, jConfiguration, key, val); done: - if (jConfiguration) - destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jConfiguration); if (ret) errno = ret; return ret; } -void hdfsConfFree(char *val) +static int hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + int ret; + jthrowable jExc = NULL; + jvalue jVal; + jstring jkey = NULL; + char buf[1024]; + + jkey = (*env)->NewStringUTF(env, key); + if (!jkey) { + (*env)->ExceptionDescribe(env); + return ENOMEM; + } + ret = invokeMethod(env, &jVal, &jExc, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (ret) { + snprintf(buf, sizeof(buf), "hadoopConfGetInt(%s)", key); + return errnoFromException(jExc, env, buf); + } + *val = jVal.i; + return 0; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jConfiguration = constructNewObjectOfClass(env, NULL, HADOOP_CONF, "()V"); + if (jConfiguration == NULL) { + fprintf(stderr, "Can't construct instance of class " + "org.apache.hadoop.conf.Configuration\n"); + ret = EINTERNAL; + goto done; + } + ret = hadoopConfGetInt(env, jConfiguration, key, val); +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) { free(val); } @@ -583,7 +631,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) } if (bld->kerbTicketCachePath) { - ret = hadoopConfSet(env, jConfiguration, + ret = hadoopConfSetStr(env, jConfiguration, KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); if (ret) goto done; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h index 5c3c6dfe2d7..4f252cd8d0d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h @@ -220,21 +220,33 @@ extern "C" { * Get a configuration string. * * @param key The key to find - * @param val (out param) The value. This will be NULL if the + * @param val (out param) The value. This will be set to NULL if the * key isn't found. You must free this string with - * hdfsConfFree. + * hdfsConfStrFree. * * @return 0 on success; nonzero error code otherwise. * Failure to find the key is not an error. */ - int hdfsConfGet(const char *key, char **val); + int hdfsConfGetStr(const char *key, char **val); /** - * Free a configuration string found with hdfsConfGet. + * Get a configuration integer. * - * @param val A configuration string obtained from hdfsConfGet + * @param key The key to find + * @param val (out param) The value. This will NOT be changed if the + * key isn't found. + * + * @return 0 on success; nonzero error code otherwise. + * Failure to find the key is not an error. */ - void hdfsConfFree(char *val); + int hdfsConfGetInt(const char *key, int32_t *val); + + /** + * Free a configuration string found with hdfsConfGetStr. + * + * @param val A configuration string obtained from hdfsConfGetStr + */ + void hdfsConfStrFree(char *val); /** * hdfsDisconnect - Disconnect from the hdfs file system. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/jni_helper.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/jni_helper.c index 97f92942f9c..d788f6282ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/jni_helper.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/jni_helper.c @@ -412,6 +412,7 @@ char *classNameOfObject(jobject jobj, JNIEnv *env) { return newstr; } + /** * Get the global JNI environemnt. * @@ -500,6 +501,11 @@ static JNIEnv* getGlobalJNIEnv(void) "with error: %d\n", rv); return NULL; } + if (invokeMethod(env, NULL, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V") != 0) { + (*env)->ExceptionDescribe(env); + } } else { //Attach this thread to the VM diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/tree.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/tree.h new file mode 100644 index 00000000000..ac78ee30af5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/tree.h @@ -0,0 +1,765 @@ +/* $NetBSD: tree.h,v 1.8 2004/03/28 19:38:30 provos Exp $ */ +/* $OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $ */ +/* $FreeBSD: src/sys/sys/tree.h,v 1.9.4.1 2011/09/23 00:51:37 kensmith Exp $ */ + +/*- + * Copyright 2002 Niels Provos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_TREE_H_ +#define _SYS_TREE_H_ + +#include + +/* + * This file defines data structures for different types of trees: + * splay trees and red-black trees. + * + * A splay tree is a self-organizing data structure. Every operation + * on the tree causes a splay to happen. The splay moves the requested + * node to the root of the tree and partly rebalances it. + * + * This has the benefit that request locality causes faster lookups as + * the requested nodes move to the top of the tree. On the other hand, + * every lookup causes memory writes. + * + * The Balance Theorem bounds the total access time for m operations + * and n inserts on an initially empty tree as O((m + n)lg n). The + * amortized cost for a sequence of m accesses to a splay tree is O(lg n); + * + * A red-black tree is a binary search tree with the node color as an + * extra attribute. It fulfills a set of conditions: + * - every search path from the root to a leaf consists of the + * same number of black nodes, + * - each red node (except for the root) has a black parent, + * - each leaf node is black. + * + * Every operation on a red-black tree is bounded as O(lg n). + * The maximum height of a red-black tree is 2lg (n+1). + */ + +#define SPLAY_HEAD(name, type) \ +struct name { \ + struct type *sph_root; /* root of the tree */ \ +} + +#define SPLAY_INITIALIZER(root) \ + { NULL } + +#define SPLAY_INIT(root) do { \ + (root)->sph_root = NULL; \ +} while (/*CONSTCOND*/ 0) + +#define SPLAY_ENTRY(type) \ +struct { \ + struct type *spe_left; /* left element */ \ + struct type *spe_right; /* right element */ \ +} + +#define SPLAY_LEFT(elm, field) (elm)->field.spe_left +#define SPLAY_RIGHT(elm, field) (elm)->field.spe_right +#define SPLAY_ROOT(head) (head)->sph_root +#define SPLAY_EMPTY(head) (SPLAY_ROOT(head) == NULL) + +/* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */ +#define SPLAY_ROTATE_RIGHT(head, tmp, field) do { \ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field); \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (/*CONSTCOND*/ 0) + +#define SPLAY_ROTATE_LEFT(head, tmp, field) do { \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field); \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (/*CONSTCOND*/ 0) + +#define SPLAY_LINKLEFT(head, tmp, field) do { \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field); \ +} while (/*CONSTCOND*/ 0) + +#define SPLAY_LINKRIGHT(head, tmp, field) do { \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field); \ +} while (/*CONSTCOND*/ 0) + +#define SPLAY_ASSEMBLE(head, node, left, right, field) do { \ + SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field); \ + SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field); \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field); \ +} while (/*CONSTCOND*/ 0) + +/* Generates prototypes and inline functions */ + +#define SPLAY_PROTOTYPE(name, type, field, cmp) \ +void name##_SPLAY(struct name *, struct type *); \ +void name##_SPLAY_MINMAX(struct name *, int); \ +struct type *name##_SPLAY_INSERT(struct name *, struct type *); \ +struct type *name##_SPLAY_REMOVE(struct name *, struct type *); \ + \ +/* Finds the node with the same key as elm */ \ +static __inline struct type * \ +name##_SPLAY_FIND(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) \ + return(NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) \ + return (head->sph_root); \ + return (NULL); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_NEXT(struct name *head, struct type *elm) \ +{ \ + name##_SPLAY(head, elm); \ + if (SPLAY_RIGHT(elm, field) != NULL) { \ + elm = SPLAY_RIGHT(elm, field); \ + while (SPLAY_LEFT(elm, field) != NULL) { \ + elm = SPLAY_LEFT(elm, field); \ + } \ + } else \ + elm = NULL; \ + return (elm); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_MIN_MAX(struct name *head, int val) \ +{ \ + name##_SPLAY_MINMAX(head, val); \ + return (SPLAY_ROOT(head)); \ +} + +/* Main splay operation. + * Moves node close to the key of elm to top + */ +#define SPLAY_GENERATE(name, type, field, cmp) \ +struct type * \ +name##_SPLAY_INSERT(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) { \ + SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL; \ + } else { \ + int __comp; \ + name##_SPLAY(head, elm); \ + __comp = (cmp)(elm, (head)->sph_root); \ + if(__comp < 0) { \ + SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\ + SPLAY_RIGHT(elm, field) = (head)->sph_root; \ + SPLAY_LEFT((head)->sph_root, field) = NULL; \ + } else if (__comp > 0) { \ + SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT(elm, field) = (head)->sph_root; \ + SPLAY_RIGHT((head)->sph_root, field) = NULL; \ + } else \ + return ((head)->sph_root); \ + } \ + (head)->sph_root = (elm); \ + return (NULL); \ +} \ + \ +struct type * \ +name##_SPLAY_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *__tmp; \ + if (SPLAY_EMPTY(head)) \ + return (NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) { \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL) { \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\ + } else { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\ + name##_SPLAY(head, elm); \ + SPLAY_RIGHT((head)->sph_root, field) = __tmp; \ + } \ + return (elm); \ + } \ + return (NULL); \ +} \ + \ +void \ +name##_SPLAY(struct name *head, struct type *elm) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ + int __comp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while ((__comp = (cmp)(elm, (head)->sph_root)) != 0) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) > 0){ \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} \ + \ +/* Splay with either the minimum or the maximum element \ + * Used to find minimum or maximum element in tree. \ + */ \ +void name##_SPLAY_MINMAX(struct name *head, int __comp) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while (1) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp > 0) { \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} + +#define SPLAY_NEGINF -1 +#define SPLAY_INF 1 + +#define SPLAY_INSERT(name, x, y) name##_SPLAY_INSERT(x, y) +#define SPLAY_REMOVE(name, x, y) name##_SPLAY_REMOVE(x, y) +#define SPLAY_FIND(name, x, y) name##_SPLAY_FIND(x, y) +#define SPLAY_NEXT(name, x, y) name##_SPLAY_NEXT(x, y) +#define SPLAY_MIN(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF)) +#define SPLAY_MAX(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_INF)) + +#define SPLAY_FOREACH(x, name, head) \ + for ((x) = SPLAY_MIN(name, head); \ + (x) != NULL; \ + (x) = SPLAY_NEXT(name, head, x)) + +/* Macros that define a red-black tree */ +#define RB_HEAD(name, type) \ +struct name { \ + struct type *rbh_root; /* root of the tree */ \ +} + +#define RB_INITIALIZER(root) \ + { NULL } + +#define RB_INIT(root) do { \ + (root)->rbh_root = NULL; \ +} while (/*CONSTCOND*/ 0) + +#define RB_BLACK 0 +#define RB_RED 1 +#define RB_ENTRY(type) \ +struct { \ + struct type *rbe_left; /* left element */ \ + struct type *rbe_right; /* right element */ \ + struct type *rbe_parent; /* parent element */ \ + int rbe_color; /* node color */ \ +} + +#define RB_LEFT(elm, field) (elm)->field.rbe_left +#define RB_RIGHT(elm, field) (elm)->field.rbe_right +#define RB_PARENT(elm, field) (elm)->field.rbe_parent +#define RB_COLOR(elm, field) (elm)->field.rbe_color +#define RB_ROOT(head) (head)->rbh_root +#define RB_EMPTY(head) (RB_ROOT(head) == NULL) + +#define RB_SET(elm, parent, field) do { \ + RB_PARENT(elm, field) = parent; \ + RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ + RB_COLOR(elm, field) = RB_RED; \ +} while (/*CONSTCOND*/ 0) + +#define RB_SET_BLACKRED(black, red, field) do { \ + RB_COLOR(black, field) = RB_BLACK; \ + RB_COLOR(red, field) = RB_RED; \ +} while (/*CONSTCOND*/ 0) + +#ifndef RB_AUGMENT +#define RB_AUGMENT(x) do {} while (0) +#endif + +#define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ + (tmp) = RB_RIGHT(elm, field); \ + if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field)) != NULL) { \ + RB_PARENT(RB_LEFT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field)) != NULL) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_LEFT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (/*CONSTCOND*/ 0) + +#define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ + (tmp) = RB_LEFT(elm, field); \ + if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field)) != NULL) { \ + RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field)) != NULL) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_RIGHT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (/*CONSTCOND*/ 0) + +/* Generates prototypes and inline functions */ +#define RB_PROTOTYPE(name, type, field, cmp) \ + RB_PROTOTYPE_INTERNAL(name, type, field, cmp,) +#define RB_PROTOTYPE_STATIC(name, type, field, cmp) \ + RB_PROTOTYPE_INTERNAL(name, type, field, cmp, __unused static) +#define RB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr) \ +attr void name##_RB_INSERT_COLOR(struct name *, struct type *); \ +attr void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\ +attr struct type *name##_RB_REMOVE(struct name *, struct type *); \ +attr struct type *name##_RB_INSERT(struct name *, struct type *); \ +attr struct type *name##_RB_FIND(struct name *, struct type *); \ +attr struct type *name##_RB_NFIND(struct name *, struct type *); \ +attr struct type *name##_RB_NEXT(struct type *); \ +attr struct type *name##_RB_PREV(struct type *); \ +attr struct type *name##_RB_MINMAX(struct name *, int); \ + \ + +/* Main rb operation. + * Moves node close to the key of elm to top + */ +#define RB_GENERATE(name, type, field, cmp) \ + RB_GENERATE_INTERNAL(name, type, field, cmp,) +#define RB_GENERATE_STATIC(name, type, field, cmp) \ + RB_GENERATE_INTERNAL(name, type, field, cmp, __unused static) +#define RB_GENERATE_INTERNAL(name, type, field, cmp, attr) \ +attr void \ +name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ +{ \ + struct type *parent, *gparent, *tmp; \ + while ((parent = RB_PARENT(elm, field)) != NULL && \ + RB_COLOR(parent, field) == RB_RED) { \ + gparent = RB_PARENT(parent, field); \ + if (parent == RB_LEFT(gparent, field)) { \ + tmp = RB_RIGHT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_RIGHT(parent, field) == elm) { \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_RIGHT(head, gparent, tmp, field); \ + } else { \ + tmp = RB_LEFT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_LEFT(parent, field) == elm) { \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_LEFT(head, gparent, tmp, field); \ + } \ + } \ + RB_COLOR(head->rbh_root, field) = RB_BLACK; \ +} \ + \ +attr void \ +name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \ +{ \ + struct type *tmp; \ + while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) && \ + elm != RB_ROOT(head)) { \ + if (RB_LEFT(parent, field) == elm) { \ + tmp = RB_RIGHT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {\ + struct type *oleft; \ + if ((oleft = RB_LEFT(tmp, field)) \ + != NULL) \ + RB_COLOR(oleft, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_RIGHT(head, tmp, oleft, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_RIGHT(tmp, field)) \ + RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } else { \ + tmp = RB_LEFT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {\ + struct type *oright; \ + if ((oright = RB_RIGHT(tmp, field)) \ + != NULL) \ + RB_COLOR(oright, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_LEFT(head, tmp, oright, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_LEFT(tmp, field)) \ + RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } \ + } \ + if (elm) \ + RB_COLOR(elm, field) = RB_BLACK; \ +} \ + \ +attr struct type * \ +name##_RB_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *child, *parent, *old = elm; \ + int color; \ + if (RB_LEFT(elm, field) == NULL) \ + child = RB_RIGHT(elm, field); \ + else if (RB_RIGHT(elm, field) == NULL) \ + child = RB_LEFT(elm, field); \ + else { \ + struct type *left; \ + elm = RB_RIGHT(elm, field); \ + while ((left = RB_LEFT(elm, field)) != NULL) \ + elm = left; \ + child = RB_RIGHT(elm, field); \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ + if (RB_PARENT(elm, field) == old) \ + parent = elm; \ + (elm)->field = (old)->field; \ + if (RB_PARENT(old, field)) { \ + if (RB_LEFT(RB_PARENT(old, field), field) == old)\ + RB_LEFT(RB_PARENT(old, field), field) = elm;\ + else \ + RB_RIGHT(RB_PARENT(old, field), field) = elm;\ + RB_AUGMENT(RB_PARENT(old, field)); \ + } else \ + RB_ROOT(head) = elm; \ + RB_PARENT(RB_LEFT(old, field), field) = elm; \ + if (RB_RIGHT(old, field)) \ + RB_PARENT(RB_RIGHT(old, field), field) = elm; \ + if (parent) { \ + left = parent; \ + do { \ + RB_AUGMENT(left); \ + } while ((left = RB_PARENT(left, field)) != NULL); \ + } \ + goto color; \ + } \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ +color: \ + if (color == RB_BLACK) \ + name##_RB_REMOVE_COLOR(head, parent, child); \ + return (old); \ +} \ + \ +/* Inserts a node into the RB tree */ \ +attr struct type * \ +name##_RB_INSERT(struct name *head, struct type *elm) \ +{ \ + struct type *tmp; \ + struct type *parent = NULL; \ + int comp = 0; \ + tmp = RB_ROOT(head); \ + while (tmp) { \ + parent = tmp; \ + comp = (cmp)(elm, parent); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + RB_SET(elm, parent, field); \ + if (parent != NULL) { \ + if (comp < 0) \ + RB_LEFT(parent, field) = elm; \ + else \ + RB_RIGHT(parent, field) = elm; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = elm; \ + name##_RB_INSERT_COLOR(head, elm); \ + return (NULL); \ +} \ + \ +/* Finds the node with the same key as elm */ \ +attr struct type * \ +name##_RB_FIND(struct name *head, struct type *elm) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + int comp; \ + while (tmp) { \ + comp = cmp(elm, tmp); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + return (NULL); \ +} \ + \ +/* Finds the first node greater than or equal to the search key */ \ +attr struct type * \ +name##_RB_NFIND(struct name *head, struct type *elm) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + struct type *res = NULL; \ + int comp; \ + while (tmp) { \ + comp = cmp(elm, tmp); \ + if (comp < 0) { \ + res = tmp; \ + tmp = RB_LEFT(tmp, field); \ + } \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + return (res); \ +} \ + \ +/* ARGSUSED */ \ +attr struct type * \ +name##_RB_NEXT(struct type *elm) \ +{ \ + if (RB_RIGHT(elm, field)) { \ + elm = RB_RIGHT(elm, field); \ + while (RB_LEFT(elm, field)) \ + elm = RB_LEFT(elm, field); \ + } else { \ + if (RB_PARENT(elm, field) && \ + (elm == RB_LEFT(RB_PARENT(elm, field), field))) \ + elm = RB_PARENT(elm, field); \ + else { \ + while (RB_PARENT(elm, field) && \ + (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\ + elm = RB_PARENT(elm, field); \ + elm = RB_PARENT(elm, field); \ + } \ + } \ + return (elm); \ +} \ + \ +/* ARGSUSED */ \ +attr struct type * \ +name##_RB_PREV(struct type *elm) \ +{ \ + if (RB_LEFT(elm, field)) { \ + elm = RB_LEFT(elm, field); \ + while (RB_RIGHT(elm, field)) \ + elm = RB_RIGHT(elm, field); \ + } else { \ + if (RB_PARENT(elm, field) && \ + (elm == RB_RIGHT(RB_PARENT(elm, field), field))) \ + elm = RB_PARENT(elm, field); \ + else { \ + while (RB_PARENT(elm, field) && \ + (elm == RB_LEFT(RB_PARENT(elm, field), field)))\ + elm = RB_PARENT(elm, field); \ + elm = RB_PARENT(elm, field); \ + } \ + } \ + return (elm); \ +} \ + \ +attr struct type * \ +name##_RB_MINMAX(struct name *head, int val) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + struct type *parent = NULL; \ + while (tmp) { \ + parent = tmp; \ + if (val < 0) \ + tmp = RB_LEFT(tmp, field); \ + else \ + tmp = RB_RIGHT(tmp, field); \ + } \ + return (parent); \ +} + +#define RB_NEGINF -1 +#define RB_INF 1 + +#define RB_INSERT(name, x, y) name##_RB_INSERT(x, y) +#define RB_REMOVE(name, x, y) name##_RB_REMOVE(x, y) +#define RB_FIND(name, x, y) name##_RB_FIND(x, y) +#define RB_NFIND(name, x, y) name##_RB_NFIND(x, y) +#define RB_NEXT(name, x, y) name##_RB_NEXT(y) +#define RB_PREV(name, x, y) name##_RB_PREV(y) +#define RB_MIN(name, x) name##_RB_MINMAX(x, RB_NEGINF) +#define RB_MAX(name, x) name##_RB_MINMAX(x, RB_INF) + +#define RB_FOREACH(x, name, head) \ + for ((x) = RB_MIN(name, head); \ + (x) != NULL; \ + (x) = name##_RB_NEXT(x)) + +#define RB_FOREACH_FROM(x, name, y) \ + for ((x) = (y); \ + ((x) != NULL) && ((y) = name##_RB_NEXT(x), (x) != NULL); \ + (x) = (y)) + +#define RB_FOREACH_SAFE(x, name, head, y) \ + for ((x) = RB_MIN(name, head); \ + ((x) != NULL) && ((y) = name##_RB_NEXT(x), (x) != NULL); \ + (x) = (y)) + +#define RB_FOREACH_REVERSE(x, name, head) \ + for ((x) = RB_MAX(name, head); \ + (x) != NULL; \ + (x) = name##_RB_PREV(x)) + +#define RB_FOREACH_REVERSE_FROM(x, name, y) \ + for ((x) = (y); \ + ((x) != NULL) && ((y) = name##_RB_PREV(x), (x) != NULL); \ + (x) = (y)) + +#define RB_FOREACH_REVERSE_SAFE(x, name, head, y) \ + for ((x) = RB_MAX(name, head); \ + ((x) != NULL) && ((y) = name##_RB_PREV(x), (x) != NULL); \ + (x) = (y)) + +#endif /* _SYS_TREE_H_ */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 27f3b9e522f..00c5e8bc446 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -909,4 +909,25 @@ + + hadoop.fuse.connection.timeout + 300 + + The minimum number of seconds that we'll cache libhdfs connection objects + in fuse_dfs. Lower values will result in lower memory consumption; higher + values may speed up access by avoiding the overhead of creating new + connection objects. + + + + + hadoop.fuse.timer.period + 5 + + The number of seconds between cache expiry checks in fuse_dfs. Lower values + will result in fuse_dfs noticing changes to Kerberos ticket caches more + quickly. + + + From d45922de2c5645e11339b94e4c31935ead66fefc Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 20 Jul 2012 20:18:48 +0000 Subject: [PATCH 03/39] svn merge --change -1363454 for reverting MAPREDUCE-4423 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1363935 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 - .../hadoop/mapreduce/task/reduce/Fetcher.java | 62 ++++----- .../mapreduce/task/reduce/TestFetcher.java | 121 ------------------ 3 files changed, 22 insertions(+), 164 deletions(-) delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5df9c136f7b..8639a6f7625 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -739,9 +739,6 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't init. (Jason Lowe via daryn) - MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans - via tgraves) - Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java index 7852da93a52..f3e7fd61c27 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java @@ -49,8 +49,7 @@ import org.apache.hadoop.mapreduce.task.reduce.MapOutput.Type; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; -import com.google.common.annotations.VisibleForTesting; - +@SuppressWarnings({"deprecation"}) class Fetcher extends Thread { private static final Log LOG = LogFactory.getLog(Fetcher.class); @@ -176,18 +175,13 @@ class Fetcher extends Thread { } } - @VisibleForTesting - protected HttpURLConnection openConnection(URL url) throws IOException { - return (HttpURLConnection)url.openConnection(); - } - /** * The crux of the matter... * * @param host {@link MapHost} from which we need to * shuffle available map-outputs. */ - protected void copyFromHost(MapHost host) throws IOException { + private void copyFromHost(MapHost host) throws IOException { // Get completed maps on 'host' List maps = scheduler.getMapsForHost(host); @@ -197,11 +191,9 @@ class Fetcher extends Thread { return; } - if(LOG.isDebugEnabled()) { - LOG.debug("Fetcher " + id + " going to fetch from " + host); - for (TaskAttemptID tmp: maps) { - LOG.debug(tmp); - } + LOG.debug("Fetcher " + id + " going to fetch from " + host); + for (TaskAttemptID tmp: maps) { + LOG.debug(tmp); } // List of maps to be fetched yet @@ -213,7 +205,7 @@ class Fetcher extends Thread { try { URL url = getMapOutputURL(host, maps); - HttpURLConnection connection = openConnection(url); + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); // generate hash of the url String msgToEncode = SecureShuffleUtils.buildMsgFrom(url); @@ -274,24 +266,17 @@ class Fetcher extends Thread { try { // Loop through available map-outputs and fetch them - // On any error, faildTasks is not null and we exit - // after putting back the remaining maps to the - // yet_to_be_fetched list and marking the failed tasks. - TaskAttemptID[] failedTasks = null; - while (!remaining.isEmpty() && failedTasks == null) { - failedTasks = copyMapOutput(host, input, remaining); - } - - if(failedTasks != null) { - for(TaskAttemptID left: failedTasks) { - scheduler.copyFailed(left, host, true); - } + // On any error, good becomes false and we exit after putting back + // the remaining maps to the yet_to_be_fetched list + boolean good = true; + while (!remaining.isEmpty() && good) { + good = copyMapOutput(host, input, remaining); } IOUtils.cleanup(LOG, input); // Sanity check - if (failedTasks == null && !remaining.isEmpty()) { + if (good && !remaining.isEmpty()) { throw new IOException("server didn't return all expected map outputs: " + remaining.size() + " left."); } @@ -300,9 +285,10 @@ class Fetcher extends Thread { scheduler.putBackKnownMapOutput(host, left); } } - } + + } - private TaskAttemptID[] copyMapOutput(MapHost host, + private boolean copyMapOutput(MapHost host, DataInputStream input, Set remaining) { MapOutput mapOutput = null; @@ -324,15 +310,14 @@ class Fetcher extends Thread { } catch (IllegalArgumentException e) { badIdErrs.increment(1); LOG.warn("Invalid map id ", e); - //Don't know which one was bad, so consider all of them as bad - return remaining.toArray(new TaskAttemptID[remaining.size()]); + return false; } // Do some basic sanity verification if (!verifySanity(compressedLength, decompressedLength, forReduce, remaining, mapId)) { - return new TaskAttemptID[] {mapId}; + return false; } LOG.debug("header: " + mapId + ", len: " + compressedLength + @@ -344,7 +329,7 @@ class Fetcher extends Thread { // Check if we can shuffle *now* ... if (mapOutput.getType() == Type.WAIT) { LOG.info("fetcher#" + id + " - MergerManager returned Status.WAIT ..."); - return new TaskAttemptID[] {mapId}; + return false; } // Go! @@ -366,18 +351,14 @@ class Fetcher extends Thread { // Note successful shuffle remaining.remove(mapId); metrics.successFetch(); - return null; + return true; } catch (IOException ioe) { ioErrs.increment(1); if (mapId == null || mapOutput == null) { LOG.info("fetcher#" + id + " failed to read map header" + mapId + " decomp: " + decompressedLength + ", " + compressedLength, ioe); - if(mapId == null) { - return remaining.toArray(new TaskAttemptID[remaining.size()]); - } else { - return new TaskAttemptID[] {mapId}; - } + return false; } LOG.info("Failed to shuffle output of " + mapId + @@ -385,8 +366,9 @@ class Fetcher extends Thread { // Inform the shuffle-scheduler mapOutput.abort(); + scheduler.copyFailed(mapId, host, true); metrics.failedFetch(); - return new TaskAttemptID[] {mapId}; + return false; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java deleted file mode 100644 index 7f7c3f50b87..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.task.reduce; - -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.ArrayList; - -import javax.crypto.SecretKey; - -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.Counters; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; -import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; -import org.junit.Test; - -/** - * Test that the Fetcher does what we expect it to. - */ -public class TestFetcher { - - public static class FakeFetcher extends Fetcher { - - private HttpURLConnection connection; - - public FakeFetcher(JobConf job, TaskAttemptID reduceId, - ShuffleScheduler scheduler, MergeManager merger, Reporter reporter, - ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, - SecretKey jobTokenSecret, HttpURLConnection connection) { - super(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, - jobTokenSecret); - this.connection = connection; - } - - @Override - protected HttpURLConnection openConnection(URL url) throws IOException { - if(connection != null) { - return connection; - } - return super.openConnection(url); - } - } - - @SuppressWarnings("unchecked") - @Test - public void testCopyFromHostBogusHeader() throws Exception { - JobConf job = new JobConf(); - TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1"); - ShuffleScheduler ss = mock(ShuffleScheduler.class); - MergeManager mm = mock(MergeManager.class); - Reporter r = mock(Reporter.class); - ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class); - ExceptionReporter except = mock(ExceptionReporter.class); - SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0}); - HttpURLConnection connection = mock(HttpURLConnection.class); - - Counters.Counter allErrs = mock(Counters.Counter.class); - when(r.getCounter(anyString(), anyString())) - .thenReturn(allErrs); - - Fetcher underTest = new FakeFetcher(job, id, ss, mm, - r, metrics, except, key, connection); - - - MapHost host = new MapHost("localhost", "http://localhost:8080/"); - - ArrayList maps = new ArrayList(1); - TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1"); - maps.add(map1ID); - TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1"); - maps.add(map2ID); - when(ss.getMapsForHost(host)).thenReturn(maps); - - String encHash = "vFE234EIFCiBgYs2tCXY/SjT8Kg="; - String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); - - when(connection.getResponseCode()).thenReturn(200); - when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)) - .thenReturn(replyHash); - ByteArrayInputStream in = new ByteArrayInputStream( - "5 BOGUS DATA\nBOGUS DATA\nBOGUS DATA\n".getBytes()); - when(connection.getInputStream()).thenReturn(in); - - underTest.copyFromHost(host); - - verify(connection) - .addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, - encHash); - - verify(allErrs).increment(1); - verify(ss).copyFailed(map1ID, host, true); - verify(ss).copyFailed(map2ID, host, true); - } - -} From decd9e1c74fa9fd631ebbd762848632b67c3e65b Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 20 Jul 2012 20:46:25 +0000 Subject: [PATCH 04/39] Move entry for HDFS-3583 in CHANGES.txt to be under branch-2. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1363949 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7a2c6acf994..eb3e60ebe40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -109,8 +109,6 @@ Trunk (unreleased changes) HDFS-3630 Modify TestPersistBlocks to use both flush and hflush (sanjay) - HDFS-3583. Convert remaining tests to Junit4. (Andrew Wang via atm) - OPTIMIZATIONS BUG FIXES @@ -352,6 +350,8 @@ Branch-2 ( Unreleased changes ) HDFS-1249. With fuse-dfs, chown which only has owner (or only group) argument fails with Input/output error. (Colin Patrick McCabe via eli) + HDFS-3583. Convert remaining tests to Junit4. (Andrew Wang via atm) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log From 80e2b4117be43cb744cbfe9949ce6c590140fbb3 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 20 Jul 2012 20:50:30 +0000 Subject: [PATCH 05/39] HADOOP-8609. IPC server logs a useless message when shutting down socket. Contributed by Jon Zuanich. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1363950 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/ipc/Server.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index dc60a12815b..d4c4e6a07cb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -268,6 +268,9 @@ Branch-2 ( Unreleased changes ) serializer or deserializer isn't available (Madhukara Phatak via harsh) + HADOOP-8609. IPC server logs a useless message when shutting down socket. + (Jon Zuanich via atm) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index d751e551bb6..7f63d81278e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -1643,7 +1643,7 @@ public abstract class Server { if (!channel.isOpen()) return; try {socket.shutdownOutput();} catch(Exception e) { - LOG.warn("Ignoring socket shutdown exception"); + LOG.debug("Ignoring socket shutdown exception", e); } if (channel.isOpen()) { try {channel.close();} catch(Exception e) {} From 9c9f29ac7e3e3533880e0b2593866a17f10087bd Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 20 Jul 2012 23:19:23 +0000 Subject: [PATCH 06/39] MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process command-line. (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1364020 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/MiniHadoopClusterManager.java | 316 ++++++++++++++++++ .../apache/hadoop/test/MapredTestDriver.java | 3 + .../src/site/apt/CLIMiniCluster.apt.vm | 84 +++++ .../src/site/apt/index.apt.vm | 1 + 5 files changed, 407 insertions(+) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8639a6f7625..d8a430efd59 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -132,6 +132,9 @@ Branch-2 ( Unreleased changes ) NEW FEATURES + MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process + command-line. (ahmed via tucu) + IMPROVEMENTS MAPREDUCE-4157. ResourceManager should not kill apps that are well behaved diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java new file mode 100644 index 00000000000..35b5e30b4fc --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -0,0 +1,316 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRClientCluster; +import org.apache.hadoop.mapred.MiniMRClientClusterFactory; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.mortbay.util.ajax.JSON; + +/** + * This class drives the creation of a mini-cluster on the local machine. By + * default, a MiniDFSCluster and MiniMRCluster are spawned on the first + * available ports that are found. + * + * A series of command line flags controls the startup cluster options. + * + * This class can dump a Hadoop configuration and some basic metadata (in JSON) + * into a text file. + * + * To shutdown the cluster, kill the process. + */ +public class MiniHadoopClusterManager { + private static final Log LOG = LogFactory + .getLog(MiniHadoopClusterManager.class); + + private MiniMRClientCluster mr; + private MiniDFSCluster dfs; + private String writeDetails; + private int numNodeManagers; + private int numDataNodes; + private int nnPort; + private int rmPort; + private int jhsPort; + private StartupOption dfsOpts; + private boolean noDFS; + private boolean noMR; + private String fs; + private String writeConfig; + private JobConf conf; + + /** + * Creates configuration options object. + */ + @SuppressWarnings("static-access") + private Options makeOptions() { + Options options = new Options(); + options + .addOption("nodfs", false, "Don't start a mini DFS cluster") + .addOption("nomr", false, "Don't start a mini MR cluster") + .addOption("nodemanagers", true, + "How many nodemanagers to start (default 1)") + .addOption("datanodes", true, "How many datanodes to start (default 1)") + .addOption("format", false, "Format the DFS (default false)") + .addOption("nnport", true, "NameNode port (default 0--we choose)") + .addOption( + "namenode", + true, + "URL of the namenode (default " + + "is either the DFS cluster or a temporary dir)") + .addOption("rmport", true, + "ResourceManager port (default 0--we choose)") + .addOption("jhsport", true, + "JobHistoryServer port (default 0--we choose)") + .addOption( + OptionBuilder.hasArgs().withArgName("property=value") + .withDescription("Options to pass into configuration object") + .create("D")) + .addOption( + OptionBuilder.hasArg().withArgName("path").withDescription( + "Save configuration to this XML file.").create("writeConfig")) + .addOption( + OptionBuilder.hasArg().withArgName("path").withDescription( + "Write basic information to this JSON file.").create( + "writeDetails")) + .addOption( + OptionBuilder.withDescription("Prints option help.").create("help")); + return options; + } + + /** + * Main entry-point. + * + * @throws URISyntaxException + */ + public void run(String[] args) throws IOException, URISyntaxException { + if (!parseArguments(args)) { + return; + } + start(); + sleepForever(); + } + + private void sleepForever() { + while (true) { + try { + Thread.sleep(1000 * 60); + } catch (InterruptedException _) { + // nothing + } + } + } + + /** + * Starts DFS and MR clusters, as specified in member-variable options. Also + * writes out configuration and details, if requested. + * + * @throws IOException + * @throws FileNotFoundException + * @throws URISyntaxException + */ + public void start() throws IOException, FileNotFoundException, + URISyntaxException { + if (!noDFS) { + dfs = new MiniDFSCluster(nnPort, conf, numDataNodes, true, true, + dfsOpts, null, null); + LOG.info("Started MiniDFSCluster -- namenode on port " + + dfs.getNameNodePort()); + } + if (!noMR) { + if (fs == null && dfs != null) { + fs = dfs.getFileSystem().getUri().toString(); + } else if (fs == null) { + fs = "file:///tmp/minimr-" + System.nanoTime(); + } + FileSystem.setDefaultUri(conf, new URI(fs)); + // Instruct the minicluster to use fixed ports, so user will know which + // ports to use when communicating with the cluster. + conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); + conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); + conf.set(YarnConfiguration.RM_ADDRESS, MiniYARNCluster.getHostname() + + ":" + this.rmPort); + conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, MiniYARNCluster.getHostname() + + ":" + this.jhsPort); + mr = MiniMRClientClusterFactory.create(this.getClass(), numNodeManagers, + conf); + LOG.info("Started MiniMRCluster"); + } + + if (writeConfig != null) { + FileOutputStream fos = new FileOutputStream(new File(writeConfig)); + conf.writeXml(fos); + fos.close(); + } + + if (writeDetails != null) { + Map map = new TreeMap(); + if (dfs != null) { + map.put("namenode_port", dfs.getNameNodePort()); + } + if (mr != null) { + map.put("resourcemanager_port", mr.getConfig().get( + YarnConfiguration.RM_ADDRESS).split(":")[1]); + } + FileWriter fw = new FileWriter(new File(writeDetails)); + fw.write(new JSON().toJSON(map)); + fw.close(); + } + } + + /** + * Shuts down in-process clusters. + * + * @throws IOException + */ + public void stop() throws IOException { + if (mr != null) { + mr.stop(); + } + if (dfs != null) { + dfs.shutdown(); + } + } + + /** + * Parses arguments and fills out the member variables. + * + * @param args + * Command-line arguments. + * @return true on successful parse; false to indicate that the program should + * exit. + */ + private boolean parseArguments(String[] args) { + Options options = makeOptions(); + CommandLine cli; + try { + CommandLineParser parser = new GnuParser(); + cli = parser.parse(options, args); + } catch (ParseException e) { + LOG.warn("options parsing failed: " + e.getMessage()); + new HelpFormatter().printHelp("...", options); + return false; + } + + if (cli.hasOption("help")) { + new HelpFormatter().printHelp("...", options); + return false; + } + if (cli.getArgs().length > 0) { + for (String arg : cli.getArgs()) { + System.err.println("Unrecognized option: " + arg); + new HelpFormatter().printHelp("...", options); + return false; + } + } + + // MR + noMR = cli.hasOption("nomr"); + numNodeManagers = intArgument(cli, "nodemanagers", 1); + rmPort = intArgument(cli, "rmport", 0); + jhsPort = intArgument(cli, "jhsport", 0); + fs = cli.getOptionValue("namenode"); + + // HDFS + noDFS = cli.hasOption("nodfs"); + numDataNodes = intArgument(cli, "datanodes", 1); + nnPort = intArgument(cli, "nnport", 0); + dfsOpts = cli.hasOption("format") ? StartupOption.FORMAT + : StartupOption.REGULAR; + + // Runner + writeDetails = cli.getOptionValue("writeDetails"); + writeConfig = cli.getOptionValue("writeConfig"); + + // General + conf = new JobConf(); + updateConfiguration(conf, cli.getOptionValues("D")); + + return true; + } + + /** + * Updates configuration based on what's given on the command line. + * + * @param conf + * The configuration object + * @param keyvalues + * An array of interleaved key value pairs. + */ + private void updateConfiguration(JobConf conf, String[] keyvalues) { + int num_confs_updated = 0; + if (keyvalues != null) { + for (String prop : keyvalues) { + String[] keyval = prop.split("=", 2); + if (keyval.length == 2) { + conf.set(keyval[0], keyval[1]); + num_confs_updated++; + } else { + LOG.warn("Ignoring -D option " + prop); + } + } + } + LOG.info("Updated " + num_confs_updated + + " configuration settings from command line."); + } + + /** + * Extracts an integer argument with specified default value. + */ + private int intArgument(CommandLine cli, String argName, int default_) { + String o = cli.getOptionValue(argName); + if (o == null) { + return default_; + } else { + return Integer.parseInt(o); + } + } + + /** + * Starts a MiniHadoopCluster. + * + * @throws URISyntaxException + */ + public static void main(String[] args) throws IOException, URISyntaxException { + new MiniHadoopClusterManager().run(args); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java index 4ab14adba15..7355b8e9fbb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java @@ -29,6 +29,7 @@ import org.apache.hadoop.mapred.TestSequenceFileInputFormat; import org.apache.hadoop.mapred.TestTextInputFormat; import org.apache.hadoop.mapred.ThreadedMapBenchmark; import org.apache.hadoop.mapreduce.FailJob; +import org.apache.hadoop.mapreduce.MiniHadoopClusterManager; import org.apache.hadoop.mapreduce.SleepJob; import org.apache.hadoop.util.ProgramDriver; @@ -101,6 +102,8 @@ public class MapredTestDriver { "Job History Log analyzer."); pgd.addClass(SliveTest.class.getSimpleName(), SliveTest.class, "HDFS Stress Test and Live Data Verification."); + pgd.addClass("minicluster", MiniHadoopClusterManager.class, + "Single process HDFS and MR cluster."); } catch(Throwable e) { e.printStackTrace(); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm new file mode 100644 index 00000000000..957b99463f0 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm @@ -0,0 +1,84 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + Hadoop MapReduce Next Generation ${project.version} - CLI MiniCluster. + --- + --- + ${maven.build.timestamp} + +Hadoop MapReduce Next Generation - CLI MiniCluster. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0} + +* {Purpose} + + Using the CLI MiniCluster, users can simply start and stop a single-node + Hadoop cluster with a single command, and without the need to set any + environment variables or manage configuration files. The CLI MiniCluster + starts both a <<>>/<<>> & <<>> clusters. + + This is useful for cases where users want to quickly experiment with a real + Hadoop cluster or test non-Java programs that rely on significant Hadoop + functionality. + +* {Hadoop Tarball} + + You should be able to obtain the Hadoop tarball from the release. Also, you + can directly create a tarball from the source: + ++---+ +$ mvn clean install -DskipTests +$ mvn package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip ++---+ + <> You will need protoc installed of version 2.4.1 or greater. + + The tarball should be available in <<>> directory. + +* {Running the MiniCluster} + + From inside the root directory of the extracted tarball, you can start the CLI + MiniCluster using the following command: + ++---+ +$ bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${project.version}-tests.jar minicluster -rmport RM_PORT -jhsport JHS_PORT ++---+ + + In the example command above, <<>> and <<>> should be + replaced by the user's choice of these port numbers. If not specified, random + free ports will be used. + + There are a number of command line arguments that the users can use to control + which services to start, and to pass other configuration properties. + The available command line arguments: + ++---+ +$ -D Options to pass into configuration object +$ -datanodes How many datanodes to start (default 1) +$ -format Format the DFS (default false) +$ -help Prints option help. +$ -jhsport JobHistoryServer port (default 0--we choose) +$ -namenode URL of the namenode (default is either the DFS +$ cluster or a temporary dir) +$ -nnport NameNode port (default 0--we choose) +$ -nodemanagers How many nodemanagers to start (default 1) +$ -nodfs Don't start a mini DFS cluster +$ -nomr Don't start a mini MR cluster +$ -rmport ResourceManager port (default 0--we choose) +$ -writeConfig Save configuration to this XML file. +$ -writeDetails Write basic information to this JSON file. ++---+ + + To display this full list of available arguments, the user can pass the + <<<-help>>> argument to the above command. diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm index ced6c3471df..dd086474982 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm @@ -49,4 +49,5 @@ MapReduce NextGen aka YARN aka MRv2 * {{{./WebApplicationProxy.html}Web Application Proxy}} + * {{{./CLIMiniCluster.html}CLI MiniCluster}} From b7b8db51dec5e14aa13ff473b5fbc21d661ded6c Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Mon, 23 Jul 2012 16:27:59 +0000 Subject: [PATCH 07/39] HDFS-3697. Enable fadvise readahead by default. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1364698 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../src/main/resources/hdfs-default.xml | 74 +++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index eb3e60ebe40..2ea4cf39716 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -363,6 +363,8 @@ Branch-2 ( Unreleased changes ) HDFS-3110. Use directRead API to reduce the number of buffer copies in libhdfs (Henry Robinson via todd) + HDFS-3697. Enable fadvise readahead by default. (todd) + BUG FIXES HDFS-3385. The last block of INodeFileUnderConstruction is not diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index ddadbdd44f7..f97c9034e4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -74,7 +74,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY = "dfs.datanode.balance.bandwidthPerSec"; public static final long DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT = 1024*1024; public static final String DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes"; - public static final long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 0; + public static final long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 4 * 1024 * 1024; // 4MB public static final String DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY = "dfs.datanode.drop.cache.behind.writes"; public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_DEFAULT = false; public static final String DFS_DATANODE_SYNC_BEHIND_WRITES_KEY = "dfs.datanode.sync.behind.writes"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 00c5e8bc446..4815ea147ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -714,6 +714,80 @@ + + dfs.datanode.readahead.bytes + 4193404 + + While reading block files, if the Hadoop native libraries are available, + the datanode can use the posix_fadvise system call to explicitly + page data into the operating system buffer cache ahead of the current + reader's position. This can improve performance especially when + disks are highly contended. + + This configuration specifies the number of bytes ahead of the current + read position which the datanode will attempt to read ahead. This + feature may be disabled by configuring this property to 0. + + If the native libraries are not available, this configuration has no + effect. + + + + + dfs.datanode.drop.cache.behind.reads + false + + In some workloads, the data read from HDFS is known to be significantly + large enough that it is unlikely to be useful to cache it in the + operating system buffer cache. In this case, the DataNode may be + configured to automatically purge all data from the buffer cache + after it is delivered to the client. This behavior is automatically + disabled for workloads which read only short sections of a block + (e.g HBase random-IO workloads). + + This may improve performance for some workloads by freeing buffer + cache spage usage for more cacheable data. + + If the Hadoop native libraries are not available, this configuration + has no effect. + + + + + dfs.datanode.drop.cache.behind.writes + false + + In some workloads, the data written to HDFS is known to be significantly + large enough that it is unlikely to be useful to cache it in the + operating system buffer cache. In this case, the DataNode may be + configured to automatically purge all data from the buffer cache + after it is written to disk. + + This may improve performance for some workloads by freeing buffer + cache spage usage for more cacheable data. + + If the Hadoop native libraries are not available, this configuration + has no effect. + + + + + dfs.datanode.sync.behind.writes + false + + If this configuration is enabled, the datanode will instruct the + operating system to enqueue all written data to the disk immediately + after it is written. This differs from the usual OS policy which + may wait for up to 30 seconds before triggering writeback. + + This may improve performance for some workloads by smoothing the + IO profile for data written to disk. + + If the Hadoop native libraries are not available, this configuration + has no effect. + + + dfs.client.failover.max.attempts 15 From 97ed48e0357094ac43533c9e6bd05067b15010c6 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 23 Jul 2012 19:31:33 +0000 Subject: [PATCH 08/39] MAPREDUCE-3893. allow capacity scheduler configs max-apps and max-am-pct per queue (tgraves via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1364764 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 + .../CapacitySchedulerConfiguration.java | 36 ++++++++- .../scheduler/capacity/LeafQueue.java | 37 ++++++---- .../capacity/TestApplicationLimits.java | 74 ++++++++++++++++++- .../src/site/apt/CapacityScheduler.apt.vm | 17 +++-- 5 files changed, 143 insertions(+), 25 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d8a430efd59..42b1d991c6a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -742,6 +742,10 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't init. (Jason Lowe via daryn) + MAPREDUCE-3893. allow capacity scheduler configs maximum-applications and + maximum-am-resource-percent configurable on a per queue basis (tgraves via + bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 86e2dd350da..300dda5011a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -46,13 +46,21 @@ public class CapacitySchedulerConfiguration extends Configuration { @Private public static final String DOT = "."; + @Private + public static final String MAXIMUM_APPLICATIONS_SUFFIX = + "maximum-applications"; + @Private public static final String MAXIMUM_SYSTEM_APPLICATIONS = - PREFIX + "maximum-applications"; + PREFIX + MAXIMUM_APPLICATIONS_SUFFIX; + + @Private + public static final String MAXIMUM_AM_RESOURCE_SUFFIX = + "maximum-am-resource-percent"; @Private public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT = - PREFIX + "maximum-am-resource-percent"; + PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX; @Private public static final String QUEUES = "queues"; @@ -131,6 +139,30 @@ public class CapacitySchedulerConfiguration extends Configuration { return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT); } + + + /** + * Get the maximum applications per queue setting. + * @param queue name of the queue + * @return setting specified or -1 if not set + */ + public int getMaximumApplicationsPerQueue(String queue) { + int maxApplicationsPerQueue = + getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, + (int)UNDEFINED); + return maxApplicationsPerQueue; + } + + /** + * Get the maximum am resource percent per queue setting. + * @param queue name of the queue + * @return per queue setting or defaults to the global am-resource-percent + * setting if per queue setting not present + */ + public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) { + return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, + getMaximumApplicationMasterResourcePercent()); + } public float getCapacity(String queue) { float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index ebf594b33c1..cf303cba8f3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -85,7 +85,7 @@ public class LeafQueue implements CSQueue { private int maxApplications; private int maxApplicationsPerUser; - private float maxAMResourcePercent; + private float maxAMResourcePerQueuePercent; private int maxActiveApplications; // Based on absolute max capacity private int maxActiveAppsUsingAbsCap; // Based on absolute capacity private int maxActiveApplicationsPerUser; @@ -156,21 +156,25 @@ public class LeafQueue implements CSQueue { float userLimitFactor = cs.getConfiguration().getUserLimitFactor(getQueuePath()); - int maxSystemJobs = cs.getConfiguration().getMaximumSystemApplications(); - int maxApplications = (int)(maxSystemJobs * absoluteCapacity); - int maxApplicationsPerUser = + int maxApplications = cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath()); + if (maxApplications < 0) { + int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications(); + maxApplications = (int)(maxSystemApps * absoluteCapacity); + } + maxApplicationsPerUser = (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor); - this.maxAMResourcePercent = - cs.getConfiguration().getMaximumApplicationMasterResourcePercent(); + this.maxAMResourcePerQueuePercent = + cs.getConfiguration(). + getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); int maxActiveApplications = CSQueueUtils.computeMaxActiveApplications( cs.getClusterResources(), this.minimumAllocation, - maxAMResourcePercent, absoluteMaxCapacity); + maxAMResourcePerQueuePercent, absoluteMaxCapacity); this.maxActiveAppsUsingAbsCap = CSQueueUtils.computeMaxActiveApplications( cs.getClusterResources(), this.minimumAllocation, - maxAMResourcePercent, absoluteCapacity); + maxAMResourcePerQueuePercent, absoluteCapacity); int maxActiveApplicationsPerUser = CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit, userLimitFactor); @@ -265,15 +269,16 @@ public class LeafQueue implements CSQueue { "userLimitFactor = " + userLimitFactor + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = " + maxApplications + - " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" + + " [= configuredMaximumSystemApplicationsPerQueue or" + + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser + " [= (int)(maxApplications * (userLimit / 100.0f) * " + "userLimitFactor) ]" + "\n" + "maxActiveApplications = " + maxActiveApplications + " [= max(" + - "(int)ceil((clusterResourceMemory / minimumAllocation) *" + - "maxAMResourcePercent * absoluteMaxCapacity)," + + "(int)ceil((clusterResourceMemory / minimumAllocation) * " + + "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," + "1) ]" + "\n" + "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap + " [= max(" + @@ -290,7 +295,7 @@ public class LeafQueue implements CSQueue { "(clusterResourceMemory * absoluteCapacity)]" + "\n" + "absoluteUsedCapacity = " + absoluteUsedCapacity + " [= usedResourcesMemory / clusterResourceMemory]" + "\n" + - "maxAMResourcePercent = " + maxAMResourcePercent + + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent + " [= configuredMaximumAMResourcePercent ]" + "\n" + "minimumAllocationFactor = " + minimumAllocationFactor + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + @@ -1387,11 +1392,11 @@ public class LeafQueue implements CSQueue { maxActiveApplications = CSQueueUtils.computeMaxActiveApplications( clusterResource, minimumAllocation, - maxAMResourcePercent, absoluteMaxCapacity); + maxAMResourcePerQueuePercent, absoluteMaxCapacity); maxActiveAppsUsingAbsCap = - CSQueueUtils.computeMaxActiveApplications( - clusterResource, minimumAllocation, - maxAMResourcePercent, absoluteCapacity); + CSQueueUtils.computeMaxActiveApplications( + clusterResource, minimumAllocation, + maxAMResourcePerQueuePercent, absoluteCapacity); maxActiveApplicationsPerUser = CSQueueUtils.computeMaxActiveApplicationsPerUser( maxActiveAppsUsingAbsCap, userLimit, userLimitFactor); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 8876bd338dd..f2f0e8d770f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -158,7 +158,9 @@ public class TestApplicationLimits { int expectedMaxActiveApps = Math.max(1, (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * - csConf.getMaximumApplicationMasterResourcePercent() * + csConf. + getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()) * queue.getAbsoluteMaximumCapacity())); assertEquals(expectedMaxActiveApps, queue.getMaximumActiveApplications()); @@ -183,7 +185,9 @@ public class TestApplicationLimits { expectedMaxActiveApps = Math.max(1, (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * - csConf.getMaximumApplicationMasterResourcePercent() * + csConf. + getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()) * queue.getAbsoluteMaximumCapacity())); assertEquals(expectedMaxActiveApps, queue.getMaximumActiveApplications()); @@ -200,6 +204,72 @@ public class TestApplicationLimits { (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()), queue.getMetrics().getAvailableMB() ); + + // should return -1 if per queue setting not set + assertEquals((int)csConf.UNDEFINED, csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); + int expectedMaxApps = (int)(csConf.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * + queue.getAbsoluteCapacity()); + assertEquals(expectedMaxApps, queue.getMaxApplications()); + + int expectedMaxAppsPerUser = (int)(expectedMaxApps * + (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor()); + assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser()); + + // should default to global setting if per queue setting not set + assertEquals((long) csConf.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, + (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath())); + + // Change the per-queue max AM resources percentage. + csConf.setFloat( + "yarn.scheduler.capacity." + + queue.getQueuePath() + + ".maximum-am-resource-percent", + 0.5f); + // Re-create queues to get new configs. + queues = new HashMap(); + root = + CapacityScheduler.parseQueue(csContext, csConf, null, "root", + queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + clusterResource = Resources.createResource(100 * 16 * GB); + + queue = (LeafQueue)queues.get(A); + expectedMaxActiveApps = + Math.max(1, + (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * + csConf. + getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()) * + queue.getAbsoluteMaximumCapacity())); + + assertEquals((long) 0.5, + (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath())); + assertEquals(expectedMaxActiveApps, + queue.getMaximumActiveApplications()); + + // Change the per-queue max applications. + csConf.setInt( + "yarn.scheduler.capacity." + + queue.getQueuePath() + + ".maximum-applications", 9999); + // Re-create queues to get new configs. + queues = new HashMap(); + root = + CapacityScheduler.parseQueue(csContext, csConf, null, "root", + queues, queues, + CapacityScheduler.queueComparator, + CapacityScheduler.applicationComparator, + TestUtils.spyHook); + + queue = (LeafQueue)queues.get(A); + assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); + assertEquals(9999, queue.getMaxApplications()); + + expectedMaxAppsPerUser = (int)(9999 * + (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor()); + assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser()); } @Test diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm index 01a9f603761..0f57f339080 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm @@ -240,17 +240,24 @@ Hadoop MapReduce Next Generation - Capacity Scheduler *--------------------------------------+--------------------------------------+ || Property || Description | *--------------------------------------+--------------------------------------+ -| <<>> | | +| <<>> / | +| <<.maximum-applications>>> | | | | Maximum number of applications in the system which can be concurrently | | | active both running and pending. Limits on each queue are directly | | | proportional to their queue capacities and user limits. This is a | | hard limit and any applications submitted when this limit is reached will | -| | be rejected. Default is 10000.| +| | be rejected. Default is 10000. This can be set for all queues with | +| | <<>> and can also be overridden on a | +| | per queue basis by setting <<.maximum-applications>>>. | *--------------------------------------+--------------------------------------+ -| yarn.scheduler.capacity.maximum-am-resource-percent | | +| <<>> / | +| <<.maximum-am-resource-percent>>> | | | | Maximum percent of resources in the cluster which can be used to run | -| | application masters - controls number of concurrent running applications. | -| | Specified as a float - ie 0.5 = 50%. Default is 10%. | +| | application masters - controls number of concurrent active applications. Limits on each | +| | queue are directly proportional to their queue capacities and user limits. | +| | Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with | +| | <<>> and can also be overridden on a | +| | per queue basis by setting <<.maximum-am-resource-percent>>> | *--------------------------------------+--------------------------------------+ * Queue Administration & Permissions From 1fd21078d8bbc78a94530ed745d4abc4e2c4418b Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 24 Jul 2012 01:53:29 +0000 Subject: [PATCH 09/39] HDFS-3709. TestStartup tests still binding to the ephemeral port. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1364865 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/namenode/TestStartup.java | 16 +++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2ea4cf39716..54f197bb220 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -525,6 +525,8 @@ Branch-2 ( Unreleased changes ) HDFS-3608. fuse_dfs: detect changes in UID ticket cache. (Colin Patrick McCabe via atm) + HDFS-3709. TestStartup tests still binding to the ephemeral port. (eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java index 90fa4d475f3..28e22aa4be7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java @@ -443,16 +443,15 @@ public class TestStartup { private void testImageChecksum(boolean compress) throws Exception { MiniDFSCluster cluster = null; - Configuration conf = new HdfsConfiguration(); if (compress) { - conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, true); + config.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, true); } try { LOG.info("\n===========================================\n" + "Starting empty cluster"); - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(config) .numDataNodes(0) .format(true) .build(); @@ -479,7 +478,7 @@ public class TestStartup { LOG.info("\n===========================================\n" + "Starting same cluster after simulated crash"); try { - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(config) .numDataNodes(0) .format(false) .build(); @@ -507,19 +506,18 @@ public class TestStartup { FileSystem localFileSys; Path hostsFile; Path excludeFile; - Configuration conf = new HdfsConfiguration(); int HEARTBEAT_INTERVAL = 1; // heartbeat interval in seconds // Set up the hosts/exclude files. - localFileSys = FileSystem.getLocal(conf); + localFileSys = FileSystem.getLocal(config); Path workingDir = localFileSys.getWorkingDirectory(); Path dir = new Path(workingDir, "build/test/data/work-dir/restartnn"); hostsFile = new Path(dir, "hosts"); excludeFile = new Path(dir, "exclude"); // Setup conf - conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); + config.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); writeConfigFile(localFileSys, excludeFile, null); - conf.set(DFSConfigKeys.DFS_HOSTS, hostsFile.toUri().getPath()); + config.set(DFSConfigKeys.DFS_HOSTS, hostsFile.toUri().getPath()); // write into hosts file ArrayListlist = new ArrayList(); byte b[] = {127, 0, 0, 1}; @@ -529,7 +527,7 @@ public class TestStartup { int numDatanodes = 1; try { - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(config) .numDataNodes(numDatanodes).setupHostsFile(true).build(); cluster.waitActive(); From d8f39138561170d4b691a50085121123b1ebc4d9 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 24 Jul 2012 15:19:08 +0000 Subject: [PATCH 10/39] HDFS-3711. Manually convert remaining tests to JUnit4. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365119 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../content/xdocs/faultinject_framework.xml | 5 +-- .../hadoop/hdfs/TestDataTransferProtocol.java | 14 +++--- .../hadoop/hdfs/TestFileConcurrentReader.java | 33 ++++++++++---- .../apache/hadoop/hdfs/TestFileCreation.java | 3 ++ .../hadoop/hdfs/protocolPB/TestPBHelper.java | 15 +++++-- .../datanode/TestSimulatedFSDataset.java | 30 ++++++++----- .../hdfs/server/namenode/TestCheckpoint.java | 45 ++++++++++++++++--- .../server/namenode/TestNameNodeMXBean.java | 36 ++++++++------- .../namenode/TestParallelImageWrite.java | 4 +- .../server/namenode/TestNNLeaseRecovery.java | 2 +- 11 files changed, 133 insertions(+), 56 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 54f197bb220..9c002059ba0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -352,6 +352,8 @@ Branch-2 ( Unreleased changes ) HDFS-3583. Convert remaining tests to Junit4. (Andrew Wang via atm) + HDFS-3711. Manually convert remaining tests to JUnit4. (Andrew Wang via atm) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml index 9f16ff78340..1673a5e11b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml @@ -332,13 +332,12 @@ package org.apache.hadoop.fs; import org.junit.Test; import org.junit.Before; -import junit.framework.TestCase; -public class DemoFiTest extends TestCase { +public class DemoFiTest { public static final String BLOCK_RECEIVER_FAULT="hdfs.datanode.BlockReceiver"; @Override @Before - public void setUp(){ + public void setUp() { //Setting up the test's environment as required } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java index aed15d819d4..f0ff407dc2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -29,8 +33,6 @@ import java.net.Socket; import java.nio.ByteBuffer; import java.util.Random; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -67,7 +69,7 @@ import org.mockito.Mockito; * This tests data transfer protocol handling in the Datanode. It sends * various forms of wrong data and verifies that Datanode handles it well. */ -public class TestDataTransferProtocol extends TestCase { +public class TestDataTransferProtocol { private static final Log LOG = LogFactory.getLog( "org.apache.hadoop.hdfs.TestDataTransferProtocol"); @@ -205,7 +207,8 @@ public class TestDataTransferProtocol extends TestCase { } } - @Test public void testOpWrite() throws IOException { + @Test + public void testOpWrite() throws IOException { int numDataNodes = 1; Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build(); @@ -333,7 +336,8 @@ public class TestDataTransferProtocol extends TestCase { } } -@Test public void testDataTransferProtocol() throws IOException { + @Test + public void testDataTransferProtocol() throws IOException { Random random = new Random(); int oneMil = 1024*1024; Path file = new Path("dataprotocol.dat"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java index fae302d5c70..97659eeab3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java @@ -16,6 +16,10 @@ * limitations under the License. */ package org.apache.hadoop.hdfs; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.util.Arrays; @@ -37,13 +41,17 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; /** * This class tests the cases of a concurrent reads/writes to a file; * ie, one writer and one or more readers can see unfinsihed blocks */ -public class TestFileConcurrentReader extends junit.framework.TestCase { +public class TestFileConcurrentReader { private enum SyncType { SYNC, @@ -69,18 +77,16 @@ public class TestFileConcurrentReader extends junit.framework.TestCase { private FileSystem fileSystem; - @Override - protected void setUp() throws IOException { + @Before + public void setUp() throws IOException { conf = new Configuration(); init(conf); } - @Override - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { cluster.shutdown(); cluster = null; - - super.tearDown(); } private void init(Configuration conf) throws IOException { @@ -145,6 +151,7 @@ public class TestFileConcurrentReader extends junit.framework.TestCase { /** * Test that that writes to an incomplete block are available to a reader */ + @Test public void testUnfinishedBlockRead() throws IOException { // create a new file in the root, write data, do no close @@ -167,6 +174,7 @@ public class TestFileConcurrentReader extends junit.framework.TestCase { * would result in too small a buffer to do the buffer-copy needed * for partial chunks. */ + @Test public void testUnfinishedBlockPacketBufferOverrun() throws IOException { // check that / exists Path path = new Path("/"); @@ -192,6 +200,7 @@ public class TestFileConcurrentReader extends junit.framework.TestCase { // use a small block size and a large write so that DN is busy creating // new blocks. This makes it almost 100% sure we can reproduce // case of client getting a DN that hasn't yet created the blocks + @Test public void testImmediateReadOfNewFile() throws IOException { final int blockSize = 64 * 1024; @@ -268,31 +277,39 @@ public class TestFileConcurrentReader extends junit.framework.TestCase { // for some reason, using tranferTo evokes the race condition more often // so test separately + @Test public void testUnfinishedBlockCRCErrorTransferTo() throws IOException { runTestUnfinishedBlockCRCError(true, SyncType.SYNC, DEFAULT_WRITE_SIZE); } + @Test public void testUnfinishedBlockCRCErrorTransferToVerySmallWrite() throws IOException { runTestUnfinishedBlockCRCError(true, SyncType.SYNC, SMALL_WRITE_SIZE); } // fails due to issue w/append, disable + @Ignore + @Test public void _testUnfinishedBlockCRCErrorTransferToAppend() throws IOException { runTestUnfinishedBlockCRCError(true, SyncType.APPEND, DEFAULT_WRITE_SIZE); } + @Test public void testUnfinishedBlockCRCErrorNormalTransfer() throws IOException { runTestUnfinishedBlockCRCError(false, SyncType.SYNC, DEFAULT_WRITE_SIZE); } + @Test public void testUnfinishedBlockCRCErrorNormalTransferVerySmallWrite() throws IOException { runTestUnfinishedBlockCRCError(false, SyncType.SYNC, SMALL_WRITE_SIZE); } // fails due to issue w/append, disable + @Ignore + @Test public void _testUnfinishedBlockCRCErrorNormalTransferAppend() throws IOException { runTestUnfinishedBlockCRCError(false, SyncType.APPEND, DEFAULT_WRITE_SIZE); @@ -441,4 +458,4 @@ public class TestFileConcurrentReader extends junit.framework.TestCase { inputStream.close(); return numRead + startPos - 1; } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index 77eb3f86576..c3909e38139 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; +import org.junit.Ignore; import org.junit.Test; /** @@ -498,6 +499,8 @@ public class TestFileCreation { * This test is currently not triggered because more HDFS work is * is needed to handle persistent leases. */ + @Ignore + @Test public void xxxtestFileCreationNamenodeRestart() throws IOException { Configuration conf = new HdfsConfiguration(); final int MAX_IDLE_TIME = 2000; // 2s diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index c009c451b7c..ad560869e5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.protocolPB; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Arrays; @@ -80,6 +80,12 @@ import com.google.common.collect.Lists; * Tests for {@link PBHelper} */ public class TestPBHelper { + + /** + * Used for asserting equality on doubles. + */ + private static final double DELTA = 0.000001; + @Test public void testConvertNamenodeRole() { assertEquals(NamenodeRoleProto.BACKUP, @@ -284,11 +290,12 @@ public class TestPBHelper { private void compare(DatanodeInfo dn1, DatanodeInfo dn2) { assertEquals(dn1.getAdminState(), dn2.getAdminState()); assertEquals(dn1.getBlockPoolUsed(), dn2.getBlockPoolUsed()); - assertEquals(dn1.getBlockPoolUsedPercent(), dn2.getBlockPoolUsedPercent()); + assertEquals(dn1.getBlockPoolUsedPercent(), + dn2.getBlockPoolUsedPercent(), DELTA); assertEquals(dn1.getCapacity(), dn2.getCapacity()); assertEquals(dn1.getDatanodeReport(), dn2.getDatanodeReport()); assertEquals(dn1.getDfsUsed(), dn1.getDfsUsed()); - assertEquals(dn1.getDfsUsedPercent(), dn1.getDfsUsedPercent()); + assertEquals(dn1.getDfsUsedPercent(), dn1.getDfsUsedPercent(), DELTA); assertEquals(dn1.getIpAddr(), dn2.getIpAddr()); assertEquals(dn1.getHostName(), dn2.getHostName()); assertEquals(dn1.getInfoPort(), dn2.getInfoPort()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java index a9609dff9da..1277f21fef8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @@ -17,13 +17,16 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.Block; @@ -33,27 +36,23 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; import org.apache.hadoop.util.DataChecksum; +import org.junit.Before; +import org.junit.Test; /** * this class tests the methods of the SimulatedFSDataset. */ -public class TestSimulatedFSDataset extends TestCase { +public class TestSimulatedFSDataset { Configuration conf = null; static final String bpid = "BP-TEST"; static final int NUMBLOCKS = 20; static final int BLOCK_LENGTH_MULTIPLIER = 79; - @Override - protected void setUp() throws Exception { - super.setUp(); + @Before + public void setUp() throws Exception { conf = new HdfsConfiguration(); SimulatedFSDataset.setFactory(conf); } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - } long blockIdToLen(long blkid) { return blkid*BLOCK_LENGTH_MULTIPLIER; @@ -90,6 +89,7 @@ public class TestSimulatedFSDataset extends TestCase { return addSomeBlocks(fsdataset, 1); } + @Test public void testFSDatasetFactory() { final Configuration conf = new Configuration(); FsDatasetSpi.Factory f = FsDatasetSpi.Factory.getFactory(conf); @@ -102,6 +102,7 @@ public class TestSimulatedFSDataset extends TestCase { assertTrue(s.isSimulated()); } + @Test public void testGetMetaData() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); @@ -123,6 +124,7 @@ public class TestSimulatedFSDataset extends TestCase { } + @Test public void testStorageUsage() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); assertEquals(fsdataset.getDfsUsed(), 0); @@ -146,6 +148,7 @@ public class TestSimulatedFSDataset extends TestCase { assertEquals(expectedLen, lengthRead); } + @Test public void testWriteRead() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); addSomeBlocks(fsdataset); @@ -157,6 +160,7 @@ public class TestSimulatedFSDataset extends TestCase { } } + @Test public void testGetBlockReport() throws IOException { SimulatedFSDataset fsdataset = getSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid); @@ -170,6 +174,7 @@ public class TestSimulatedFSDataset extends TestCase { } } + @Test public void testInjectionEmpty() throws IOException { SimulatedFSDataset fsdataset = getSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid); @@ -198,6 +203,7 @@ public class TestSimulatedFSDataset extends TestCase { assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining()); } + @Test public void testInjectionNonEmpty() throws IOException { SimulatedFSDataset fsdataset = getSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.getBlockReport(bpid); @@ -271,6 +277,7 @@ public class TestSimulatedFSDataset extends TestCase { } } + @Test public void testInValidBlocks() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); @@ -282,6 +289,7 @@ public class TestSimulatedFSDataset extends TestCase { checkInvalidBlock(b); } + @Test public void testInvalidate() throws IOException { final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); int bytesAdded = addSomeBlocks(fsdataset); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index b2496f92b15..e505e73736e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -20,6 +20,12 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; import static org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil.assertNNHasCheckpoints; import static org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil.getNameNodeCurrentDirs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; @@ -31,8 +37,6 @@ import java.util.Collection; import java.util.List; import java.util.Random; -import junit.framework.TestCase; - import org.apache.commons.cli.ParseException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -69,6 +73,8 @@ import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; +import org.junit.Before; +import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -84,7 +90,7 @@ import com.google.common.primitives.Ints; /** * This class tests the creation and validation of a checkpoint. */ -public class TestCheckpoint extends TestCase { +public class TestCheckpoint { static { ((Log4JLogger)FSImage.LOG).getLogger().setLevel(Level.ALL); @@ -100,7 +106,7 @@ public class TestCheckpoint extends TestCase { private CheckpointFaultInjector faultInjector; - @Override + @Before public void setUp() throws IOException { FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); @@ -139,6 +145,7 @@ public class TestCheckpoint extends TestCase { /* * Verify that namenode does not startup if one namedir is bad. */ + @Test public void testNameDirError() throws IOException { LOG.info("Starting testNameDirError"); Configuration conf = new HdfsConfiguration(); @@ -180,6 +187,7 @@ public class TestCheckpoint extends TestCase { * correctly (by removing the storage directory) * See https://issues.apache.org/jira/browse/HDFS-2011 */ + @Test public void testWriteTransactionIdHandlesIOE() throws Exception { LOG.info("Check IOException handled correctly by writeTransactionIdFile"); ArrayList fsImageDirs = new ArrayList(); @@ -214,6 +222,7 @@ public class TestCheckpoint extends TestCase { /* * Simulate namenode crashing after rolling edit log. */ + @Test public void testSecondaryNamenodeError1() throws IOException { LOG.info("Starting testSecondaryNamenodeError1"); @@ -279,6 +288,7 @@ public class TestCheckpoint extends TestCase { /* * Simulate a namenode crash after uploading new image */ + @Test public void testSecondaryNamenodeError2() throws IOException { LOG.info("Starting testSecondaryNamenodeError2"); Configuration conf = new HdfsConfiguration(); @@ -340,6 +350,7 @@ public class TestCheckpoint extends TestCase { /* * Simulate a secondary namenode crash after rolling the edit log. */ + @Test public void testSecondaryNamenodeError3() throws IOException { LOG.info("Starting testSecondaryNamenodeError3"); Configuration conf = new HdfsConfiguration(); @@ -412,6 +423,7 @@ public class TestCheckpoint extends TestCase { * back to the name-node. * Used to truncate primary fsimage file. */ + @Test public void testSecondaryFailsToReturnImage() throws IOException { Mockito.doThrow(new IOException("If this exception is not caught by the " + "name-node, fs image will be truncated.")) @@ -425,6 +437,7 @@ public class TestCheckpoint extends TestCase { * before even setting the length header. This used to cause image * truncation. Regression test for HDFS-3330. */ + @Test public void testSecondaryFailsWithErrorBeforeSettingHeaders() throws IOException { Mockito.doThrow(new Error("If this exception is not caught by the " + @@ -497,6 +510,7 @@ public class TestCheckpoint extends TestCase { * The length header in the HTTP transfer should prevent * this from corrupting the NN. */ + @Test public void testNameNodeImageSendFailWrongSize() throws IOException { LOG.info("Starting testNameNodeImageSendFailWrongSize"); @@ -511,6 +525,7 @@ public class TestCheckpoint extends TestCase { * The digest header in the HTTP transfer should prevent * this from corrupting the NN. */ + @Test public void testNameNodeImageSendFailWrongDigest() throws IOException { LOG.info("Starting testNameNodeImageSendFailWrongDigest"); @@ -528,7 +543,7 @@ public class TestCheckpoint extends TestCase { private void doSendFailTest(String exceptionSubstring) throws IOException { Configuration conf = new HdfsConfiguration(); - Path file1 = new Path("checkpoint-doSendFailTest-" + getName() + ".dat"); + Path file1 = new Path("checkpoint-doSendFailTest-doSendFailTest.dat"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDatanodes) .build(); @@ -574,6 +589,7 @@ public class TestCheckpoint extends TestCase { * Test that the NN locks its storage and edits directories, and won't start up * if the directories are already locked **/ + @Test public void testNameDirLocking() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) @@ -603,6 +619,7 @@ public class TestCheckpoint extends TestCase { * Test that, if the edits dir is separate from the name dir, it is * properly locked. **/ + @Test public void testSeparateEditsDirLocking() throws IOException { Configuration conf = new HdfsConfiguration(); File editsDir = new File(MiniDFSCluster.getBaseDirectory() + @@ -638,6 +655,7 @@ public class TestCheckpoint extends TestCase { /** * Test that the SecondaryNameNode properly locks its storage directories. */ + @Test public void testSecondaryNameNodeLocking() throws Exception { // Start a primary NN so that the secondary will start successfully Configuration conf = new HdfsConfiguration(); @@ -687,6 +705,7 @@ public class TestCheckpoint extends TestCase { * Test that, an attempt to lock a storage that is already locked by a nodename, * logs error message that includes JVM name of the namenode that locked it. */ + @Test public void testStorageAlreadyLockedErrorMessage() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) @@ -763,6 +782,7 @@ public class TestCheckpoint extends TestCase { * 2. if the NN does not contain an image, importing a checkpoint * succeeds and re-saves the image */ + @Test public void testImportCheckpoint() throws Exception { Configuration conf = new HdfsConfiguration(); Path testPath = new Path("/testfile"); @@ -861,6 +881,7 @@ public class TestCheckpoint extends TestCase { /** * Tests checkpoint in HDFS. */ + @Test public void testCheckpoint() throws IOException { Path file1 = new Path("checkpoint.dat"); Path file2 = new Path("checkpoint2.dat"); @@ -951,6 +972,7 @@ public class TestCheckpoint extends TestCase { /** * Tests save namespace. */ + @Test public void testSaveNamespace() throws IOException { MiniDFSCluster cluster = null; DistributedFileSystem fs = null; @@ -1057,6 +1079,7 @@ public class TestCheckpoint extends TestCase { /* Test case to test CheckpointSignature */ @SuppressWarnings("deprecation") + @Test public void testCheckpointSignature() throws IOException { MiniDFSCluster cluster = null; @@ -1091,6 +1114,7 @@ public class TestCheckpoint extends TestCase { * - it then fails again for the same reason * - it then tries to checkpoint a third time */ + @Test public void testCheckpointAfterTwoFailedUploads() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1147,6 +1171,7 @@ public class TestCheckpoint extends TestCase { * * @throws IOException */ + @Test public void testMultipleSecondaryNamenodes() throws IOException { Configuration conf = new HdfsConfiguration(); String nameserviceId1 = "ns1"; @@ -1197,6 +1222,7 @@ public class TestCheckpoint extends TestCase { * Test that the secondary doesn't have to re-download image * if it hasn't changed. */ + @Test public void testSecondaryImageDownload() throws IOException { LOG.info("Starting testSecondaryImageDownload"); Configuration conf = new HdfsConfiguration(); @@ -1279,6 +1305,7 @@ public class TestCheckpoint extends TestCase { * It verifies that this works even though the earlier-txid checkpoint gets * uploaded after the later-txid checkpoint. */ + @Test public void testMultipleSecondaryNNsAgainstSameNN() throws Exception { Configuration conf = new HdfsConfiguration(); @@ -1364,6 +1391,7 @@ public class TestCheckpoint extends TestCase { * It verifies that one of the two gets an error that it's uploading a * duplicate checkpoint, and the other one succeeds. */ + @Test public void testMultipleSecondaryNNsAgainstSameNN2() throws Exception { Configuration conf = new HdfsConfiguration(); @@ -1457,6 +1485,7 @@ public class TestCheckpoint extends TestCase { * is running. The secondary should shut itself down if if talks to a NN * with the wrong namespace. */ + @Test public void testReformatNNBetweenCheckpoints() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1514,6 +1543,7 @@ public class TestCheckpoint extends TestCase { * Test that the primary NN will not serve any files to a 2NN who doesn't * share its namespace ID, and also will not accept any files from one. */ + @Test public void testNamespaceVerifiedOnFileTransfer() throws IOException { MiniDFSCluster cluster = null; @@ -1575,6 +1605,7 @@ public class TestCheckpoint extends TestCase { * the non-failed storage directory receives the checkpoint. */ @SuppressWarnings("deprecation") + @Test public void testCheckpointWithFailedStorageDir() throws Exception { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1639,6 +1670,7 @@ public class TestCheckpoint extends TestCase { * @throws Exception */ @SuppressWarnings("deprecation") + @Test public void testCheckpointWithSeparateDirsAfterNameFails() throws Exception { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1711,6 +1743,7 @@ public class TestCheckpoint extends TestCase { /** * Test that the 2NN triggers a checkpoint after the configurable interval */ + @Test public void testCheckpointTriggerOnTxnCount() throws Exception { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1764,6 +1797,7 @@ public class TestCheckpoint extends TestCase { * logs that connect the 2NN's old checkpoint to the current txid * get archived. Then, the 2NN tries to checkpoint again. */ + @Test public void testSecondaryHasVeryOutOfDateImage() throws IOException { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1801,6 +1835,7 @@ public class TestCheckpoint extends TestCase { } } + @Test public void testCommandLineParsing() throws ParseException { SecondaryNameNode.CommandLineOpts opts = new SecondaryNameNode.CommandLineOpts(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index ab013b5fbfb..cf624b75aad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -29,8 +29,6 @@ import java.util.Map; import javax.management.MBeanServer; import javax.management.ObjectName; -import junit.framework.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -42,6 +40,12 @@ import org.mortbay.util.ajax.JSON; * Class for testing {@link NameNodeMXBean} implementation */ public class TestNameNodeMXBean { + + /** + * Used to assert equality between doubles + */ + private static final double DELTA = 0.000001; + @SuppressWarnings({ "unchecked", "deprecation" }) @Test public void testNameNodeMXBeanInfo() throws Exception { @@ -59,36 +63,36 @@ public class TestNameNodeMXBean { "Hadoop:service=NameNode,name=NameNodeInfo"); // get attribute "ClusterId" String clusterId = (String) mbs.getAttribute(mxbeanName, "ClusterId"); - Assert.assertEquals(fsn.getClusterId(), clusterId); + assertEquals(fsn.getClusterId(), clusterId); // get attribute "BlockPoolId" String blockpoolId = (String) mbs.getAttribute(mxbeanName, "BlockPoolId"); - Assert.assertEquals(fsn.getBlockPoolId(), blockpoolId); + assertEquals(fsn.getBlockPoolId(), blockpoolId); // get attribute "Version" String version = (String) mbs.getAttribute(mxbeanName, "Version"); - Assert.assertEquals(fsn.getVersion(), version); - Assert.assertTrue(version.equals(VersionInfo.getVersion() + assertEquals(fsn.getVersion(), version); + assertTrue(version.equals(VersionInfo.getVersion() + ", r" + VersionInfo.getRevision())); // get attribute "Used" Long used = (Long) mbs.getAttribute(mxbeanName, "Used"); - Assert.assertEquals(fsn.getUsed(), used.longValue()); + assertEquals(fsn.getUsed(), used.longValue()); // get attribute "Total" Long total = (Long) mbs.getAttribute(mxbeanName, "Total"); - Assert.assertEquals(fsn.getTotal(), total.longValue()); + assertEquals(fsn.getTotal(), total.longValue()); // get attribute "safemode" String safemode = (String) mbs.getAttribute(mxbeanName, "Safemode"); - Assert.assertEquals(fsn.getSafemode(), safemode); + assertEquals(fsn.getSafemode(), safemode); // get attribute nondfs Long nondfs = (Long) (mbs.getAttribute(mxbeanName, "NonDfsUsedSpace")); - Assert.assertEquals(fsn.getNonDfsUsedSpace(), nondfs.longValue()); + assertEquals(fsn.getNonDfsUsedSpace(), nondfs.longValue()); // get attribute percentremaining Float percentremaining = (Float) (mbs.getAttribute(mxbeanName, "PercentRemaining")); - Assert.assertEquals(fsn.getPercentRemaining(), percentremaining - .floatValue()); + assertEquals(fsn.getPercentRemaining(), percentremaining + .floatValue(), DELTA); // get attribute Totalblocks Long totalblocks = (Long) (mbs.getAttribute(mxbeanName, "TotalBlocks")); - Assert.assertEquals(fsn.getTotalBlocks(), totalblocks.longValue()); + assertEquals(fsn.getTotalBlocks(), totalblocks.longValue()); // get attribute alivenodeinfo String alivenodeinfo = (String) (mbs.getAttribute(mxbeanName, "LiveNodes")); @@ -103,15 +107,15 @@ public class TestNameNodeMXBean { assertTrue(liveNode.containsKey("numBlocks")); assertTrue(((Long)liveNode.get("numBlocks")) == 0); } - Assert.assertEquals(fsn.getLiveNodes(), alivenodeinfo); + assertEquals(fsn.getLiveNodes(), alivenodeinfo); // get attribute deadnodeinfo String deadnodeinfo = (String) (mbs.getAttribute(mxbeanName, "DeadNodes")); - Assert.assertEquals(fsn.getDeadNodes(), deadnodeinfo); + assertEquals(fsn.getDeadNodes(), deadnodeinfo); // get attribute NameDirStatuses String nameDirStatuses = (String) (mbs.getAttribute(mxbeanName, "NameDirStatuses")); - Assert.assertEquals(fsn.getNameDirStatuses(), nameDirStatuses); + assertEquals(fsn.getNameDirStatuses(), nameDirStatuses); Map> statusMap = (Map>) JSON.parse(nameDirStatuses); Collection nameDirUris = cluster.getNameDirs(0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestParallelImageWrite.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestParallelImageWrite.java index f9ba34e15f4..420026103e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestParallelImageWrite.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestParallelImageWrite.java @@ -26,8 +26,6 @@ import java.io.File; import java.util.Collections; import java.util.List; -import junit.framework.AssertionFailedError; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -132,7 +130,7 @@ public class TestParallelImageWrite { * @param fsn - the FSNamesystem being checked. * @param numImageDirs - the configured number of StorageDirectory of type IMAGE. * @return - the md5 hash of the most recent FSImage files, which must all be the same. - * @throws AssertionFailedError if image files are empty or different, + * @throws AssertionError if image files are empty or different, * if less than two StorageDirectory are provided, or if the * actual number of StorageDirectory is less than configured. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java index ba76a7bd19e..089cf7b92b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hdfs.server.namenode; -import static junit.framework.Assert.assertTrue; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; From 3ef19e9dbd5995634aac9b4011765e1d889ea6f5 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Tue, 24 Jul 2012 17:33:15 +0000 Subject: [PATCH 11/39] MAPREDUCE-4438. Add a simple, generic client to run 'easy' AMs in YARN. Contributed by Bikas Saha. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365185 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../distributedshell/ApplicationMaster.java | 5 +- .../pom.xml | 110 +++++ .../UnmanagedAMLauncher.java | 405 ++++++++++++++++++ .../TestUnmanagedAMLauncher.java | 163 +++++++ .../src/test/resources/yarn-site.xml | 21 + .../hadoop-yarn-applications/pom.xml | 1 + 7 files changed, 707 insertions(+), 1 deletion(-) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/yarn-site.xml diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 42b1d991c6a..18863a1a4a0 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -161,6 +161,9 @@ Release 2.1.0-alpha - Unreleased MAPREDUCE-3451. Port Fair Scheduler to MR2 (pwendell via tucu) + MAPREDUCE-4438. Add a simple, generic client to run 'easy' AMs in YARN. + (Bikas Saha via acmurthy) + IMPROVEMENTS MAPREDUCE-4440. Changed SchedulerApp and SchedulerNode to be a minimal diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 592ab03822a..4323962e807 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -290,7 +290,10 @@ public class ApplicationMaster { Map envs = System.getenv(); appAttemptID = Records.newRecord(ApplicationAttemptId.class); - if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { + if (envs.containsKey(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)) { + appAttemptID = ConverterUtils.toApplicationAttemptId(envs + .get(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)); + } else if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml new file mode 100644 index 00000000000..0dd77604ba9 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml @@ -0,0 +1,110 @@ + + + + + hadoop-yarn-applications + org.apache.hadoop + 3.0.0-SNAPSHOT + + 4.0.0 + org.apache.hadoop + hadoop-yarn-applications-unmanaged-am-launcher + 3.0.0-SNAPSHOT + hadoop-yarn-applications-unmanaged-am-launcher + + + + ${project.parent.parent.basedir} + + + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-yarn-server-nodemanager + test + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + test + + + org.apache.hadoop + hadoop-yarn-server-common + test + + + org.apache.hadoop + hadoop-mapreduce-client-core + test + + + org.apache.hadoop + hadoop-yarn-applications-distributedshell + 3.0.0-SNAPSHOT + test + + + org.apache.hadoop + hadoop-yarn-server-tests + test-jar + test + + + + + + + maven-dependency-plugin + + + build-classpath + generate-sources + + build-classpath + + + + target/classes/yarn-apps-am-generated-classpath + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${java.home} + + + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java new file mode 100644 index 00000000000..cd5f94e544e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -0,0 +1,405 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.applications.unmanagedamlauncher; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRemoteException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.util.Records; + +/** + * The UnmanagedLauncher is a simple client that launches and unmanaged AM. An + * unmanagedAM is an AM that is not launched and managed by the RM. The client + * creates a new application on the RM and negotiates a new attempt id. Then it + * waits for the RM app state to reach be YarnApplicationState.ACCEPTED after + * which it spawns the AM in another process and passes it the attempt id via + * env variable ApplicationConstants.AM_APP_ATTEMPT_ID_ENV. The AM can be in any + * language. The AM can register with the RM using the attempt id and proceed as + * normal. The client redirects app stdout and stderr to its own stdout and + * stderr and waits for the AM process to exit. Then it waits for the RM to + * report app completion. + */ +public class UnmanagedAMLauncher { + private static final Log LOG = LogFactory.getLog(UnmanagedAMLauncher.class); + + private Configuration conf; + + // RPC to communicate to RM + private YarnRPC rpc; + + // Handle to talk to the Resource Manager/Applications Manager + private ClientRMProtocol rmClient; + + // Application master specific info to register a new Application with RM/ASM + private String appName = ""; + // App master priority + private int amPriority = 0; + // Queue for App master + private String amQueue = ""; + // cmd to start AM + private String amCmd = null; + // set the classpath explicitly + private String classpath = null; + + /** + * @param args + * Command line arguments + */ + public static void main(String[] args) { + try { + UnmanagedAMLauncher client = new UnmanagedAMLauncher(); + LOG.info("Initializing Client"); + boolean doRun = client.init(args); + if (!doRun) { + System.exit(0); + } + client.run(); + } catch (Throwable t) { + LOG.fatal("Error running Client", t); + System.exit(1); + } + } + + /** + */ + public UnmanagedAMLauncher(Configuration conf) throws Exception { + // Set up RPC + this.conf = conf; + rpc = YarnRPC.create(conf); + } + + public UnmanagedAMLauncher() throws Exception { + this(new Configuration()); + } + + private void printUsage(Options opts) { + new HelpFormatter().printHelp("Client", opts); + } + + public boolean init(String[] args) throws ParseException { + + Options opts = new Options(); + opts.addOption("appname", true, + "Application Name. Default value - UnmanagedAM"); + opts.addOption("priority", true, "Application Priority. Default 0"); + opts.addOption("queue", true, + "RM Queue in which this application is to be submitted"); + opts.addOption("master_memory", true, + "Amount of memory in MB to be requested to run the application master"); + opts.addOption("cmd", true, "command to start unmanaged AM (required)"); + opts.addOption("classpath", true, "additional classpath"); + opts.addOption("help", false, "Print usage"); + CommandLine cliParser = new GnuParser().parse(opts, args); + + if (args.length == 0) { + printUsage(opts); + throw new IllegalArgumentException( + "No args specified for client to initialize"); + } + + if (cliParser.hasOption("help")) { + printUsage(opts); + return false; + } + + appName = cliParser.getOptionValue("appname", "UnmanagedAM"); + amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); + amQueue = cliParser.getOptionValue("queue", ""); + classpath = cliParser.getOptionValue("classpath", null); + + amCmd = cliParser.getOptionValue("cmd"); + if (amCmd == null) { + printUsage(opts); + throw new IllegalArgumentException( + "No cmd specified for application master"); + } + + return true; + } + + private void connectToRM() throws IOException { + YarnConfiguration yarnConf = new YarnConfiguration(conf); + InetSocketAddress rmAddress = yarnConf.getSocketAddr( + YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_PORT); + LOG.info("Connecting to ResourceManager at " + rmAddress); + rmClient = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, + rmAddress, conf)); + } + + private GetNewApplicationResponse getApplication() throws YarnRemoteException { + GetNewApplicationRequest request = Records + .newRecord(GetNewApplicationRequest.class); + GetNewApplicationResponse response = rmClient.getNewApplication(request); + LOG.info("Got new application id=" + response.getApplicationId()); + return response; + } + + public void launchAM(ApplicationAttemptId attemptId) throws IOException { + Map env = System.getenv(); + ArrayList envAMList = new ArrayList(); + boolean setClasspath = false; + for (Map.Entry entry : env.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if(key.equals("CLASSPATH")) { + setClasspath = true; + if(classpath != null) { + value = value + File.pathSeparator + classpath; + } + } + envAMList.add(key + "=" + value); + } + + if(!setClasspath && classpath!=null) { + envAMList.add("CLASSPATH="+classpath); + } + + envAMList.add(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV + "=" + attemptId); + + String[] envAM = new String[envAMList.size()]; + Process amProc = Runtime.getRuntime().exec(amCmd, envAMList.toArray(envAM)); + + final BufferedReader errReader = + new BufferedReader(new InputStreamReader(amProc + .getErrorStream())); + final BufferedReader inReader = + new BufferedReader(new InputStreamReader(amProc + .getInputStream())); + + // read error and input streams as this would free up the buffers + // free the error stream buffer + Thread errThread = new Thread() { + @Override + public void run() { + try { + String line = errReader.readLine(); + while((line != null) && !isInterrupted()) { + System.err.println(line); + line = errReader.readLine(); + } + } catch(IOException ioe) { + LOG.warn("Error reading the error stream", ioe); + } + } + }; + Thread outThread = new Thread() { + @Override + public void run() { + try { + String line = inReader.readLine(); + while((line != null) && !isInterrupted()) { + System.out.println(line); + line = inReader.readLine(); + } + } catch(IOException ioe) { + LOG.warn("Error reading the out stream", ioe); + } + } + }; + try { + errThread.start(); + outThread.start(); + } catch (IllegalStateException ise) { } + + // wait for the process to finish and check the exit code + try { + int exitCode = amProc.waitFor(); + LOG.info("AM process exited with value: " + exitCode); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + try { + // make sure that the error thread exits + // on Windows these threads sometimes get stuck and hang the execution + // timeout and join later after destroying the process. + errThread.join(); + outThread.join(); + errReader.close(); + inReader.close(); + } catch (InterruptedException ie) { + LOG.info("ShellExecutor: Interrupted while reading the error/out stream", + ie); + } catch (IOException ioe) { + LOG.warn("Error while closing the error/out stream", ioe); + } + amProc.destroy(); + } + + public boolean run() throws IOException { + LOG.info("Starting Client"); + + // Connect to ResourceManager + connectToRM(); + assert (rmClient != null); + + // Get a new application id + GetNewApplicationResponse newApp = getApplication(); + ApplicationId appId = newApp.getApplicationId(); + + // Create launch context for app master + LOG.info("Setting up application submission context for ASM"); + ApplicationSubmissionContext appContext = Records + .newRecord(ApplicationSubmissionContext.class); + + // set the application id + appContext.setApplicationId(appId); + // set the application name + appContext.setApplicationName(appName); + + // Set the priority for the application master + Priority pri = Records.newRecord(Priority.class); + pri.setPriority(amPriority); + appContext.setPriority(pri); + + // Set the queue to which this application is to be submitted in the RM + appContext.setQueue(amQueue); + + // Set up the container launch context for the application master + ContainerLaunchContext amContainer = Records + .newRecord(ContainerLaunchContext.class); + appContext.setAMContainerSpec(amContainer); + + // unmanaged AM + appContext.setUnmanagedAM(true); + LOG.info("Setting unmanaged AM"); + + // Create the request to send to the applications manager + SubmitApplicationRequest appRequest = Records + .newRecord(SubmitApplicationRequest.class); + appRequest.setApplicationSubmissionContext(appContext); + + // Submit the application to the applications manager + LOG.info("Submitting application to ASM"); + rmClient.submitApplication(appRequest); + + // Monitor the application to wait for launch state + ApplicationReport appReport = monitorApplication(appId, + EnumSet.of(YarnApplicationState.ACCEPTED)); + ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); + LOG.info("Launching application with id: " + attemptId); + + // launch AM + launchAM(attemptId); + + // Monitor the application for end state + appReport = monitorApplication(appId, EnumSet.of( + YarnApplicationState.KILLED, YarnApplicationState.FAILED, + YarnApplicationState.FINISHED)); + YarnApplicationState appState = appReport.getYarnApplicationState(); + FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus(); + + LOG.info("App ended with state: " + appReport.getYarnApplicationState() + + " and status: " + appStatus); + if (YarnApplicationState.FINISHED == appState + && FinalApplicationStatus.SUCCEEDED == appStatus) { + LOG.info("Application has completed successfully."); + return true; + } else { + LOG.info("Application did finished unsuccessfully." + " YarnState=" + + appState.toString() + ", FinalStatus=" + appStatus.toString()); + return false; + } + } + + /** + * Monitor the submitted application for completion. Kill application if time + * expires. + * + * @param appId + * Application Id of application to be monitored + * @return true if application completed successfully + * @throws YarnRemoteException + */ + private ApplicationReport monitorApplication(ApplicationId appId, + Set finalState) throws YarnRemoteException { + + while (true) { + + // Check app status every 1 second. + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + LOG.debug("Thread sleep in monitoring loop interrupted"); + } + + // Get application report for the appId we are interested in + GetApplicationReportRequest reportRequest = Records + .newRecord(GetApplicationReportRequest.class); + reportRequest.setApplicationId(appId); + GetApplicationReportResponse reportResponse = rmClient + .getApplicationReport(reportRequest); + ApplicationReport report = reportResponse.getApplicationReport(); + + LOG.info("Got application report from ASM for" + ", appId=" + + appId.getId() + ", appAttemptId=" + + report.getCurrentApplicationAttemptId() + ", clientToken=" + + report.getClientToken() + ", appDiagnostics=" + + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + + ", yarnAppState=" + report.getYarnApplicationState().toString() + + ", distributedFinalState=" + + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + + report.getTrackingUrl() + ", appUser=" + report.getUser()); + + YarnApplicationState state = report.getYarnApplicationState(); + if (finalState.contains(state)) { + return report; + } + + } + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java new file mode 100644 index 00000000000..0c417bb7cba --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.applications.unmanagedamlauncher; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.URL; + +import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestUnmanagedAMLauncher { + + private static final Log LOG = LogFactory + .getLog(TestUnmanagedAMLauncher.class); + + protected static MiniYARNCluster yarnCluster = null; + protected static Configuration conf = new Configuration(); + + @BeforeClass + public static void setup() throws InterruptedException, IOException { + LOG.info("Starting up YARN cluster"); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + if (yarnCluster == null) { + yarnCluster = new MiniYARNCluster( + TestUnmanagedAMLauncher.class.getName(), 1, 1, 1); + yarnCluster.init(conf); + yarnCluster.start(); + URL url = Thread.currentThread().getContextClassLoader() + .getResource("yarn-site.xml"); + if (url == null) { + throw new RuntimeException( + "Could not find 'yarn-site.xml' dummy file in classpath"); + } + OutputStream os = new FileOutputStream(new File(url.getPath())); + yarnCluster.getConfig().writeXml(os); + os.close(); + } + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws IOException { + if (yarnCluster != null) { + yarnCluster.stop(); + yarnCluster = null; + } + } + + private static String getTestRuntimeClasspath() { + + InputStream classpathFileStream = null; + BufferedReader reader = null; + String envClassPath = ""; + + LOG.info("Trying to generate classpath for app master from current thread's classpath"); + try { + + // Create classpath from generated classpath + // Check maven pom.xml for generated classpath info + // Works if compile time env is same as runtime. Mainly tests. + ClassLoader thisClassLoader = Thread.currentThread() + .getContextClassLoader(); + String generatedClasspathFile = "yarn-apps-am-generated-classpath"; + classpathFileStream = thisClassLoader + .getResourceAsStream(generatedClasspathFile); + if (classpathFileStream == null) { + LOG.info("Could not classpath resource from class loader"); + return envClassPath; + } + LOG.info("Readable bytes from stream=" + classpathFileStream.available()); + reader = new BufferedReader(new InputStreamReader(classpathFileStream)); + String cp = reader.readLine(); + if (cp != null) { + envClassPath += cp.trim() + File.pathSeparator; + } + // yarn-site.xml at this location contains proper config for mini cluster + URL url = thisClassLoader.getResource("yarn-site.xml"); + envClassPath += new File(url.getFile()).getParent(); + } catch (IOException e) { + LOG.info("Could not find the necessary resource to generate class path for tests. Error=" + + e.getMessage()); + } + + try { + if (classpathFileStream != null) { + classpathFileStream.close(); + } + if (reader != null) { + reader.close(); + } + } catch (IOException e) { + LOG.info("Failed to close class path file stream or reader. Error=" + + e.getMessage()); + } + return envClassPath; + } + + @Test + public void testDSShell() throws Exception { + String classpath = getTestRuntimeClasspath(); + String javaHome = System.getenv("JAVA_HOME"); + if (javaHome == null) { + LOG.fatal("JAVA_HOME not defined. Test not running."); + return; + } + // start dist-shell with 0 containers because container launch will fail if + // there are no dist cache resources. + String[] args = { + "--classpath", + classpath, + "--cmd", + javaHome + + "/bin/java -Xmx512m " + + "org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster " + + "--container_memory 128 --num_containers 0 --priority 0 --shell_command ls" }; + + LOG.info("Initializing Launcher"); + UnmanagedAMLauncher launcher = new UnmanagedAMLauncher(new Configuration( + yarnCluster.getConfig())); + boolean initSuccess = launcher.init(args); + Assert.assertTrue(initSuccess); + LOG.info("Running Launcher"); + boolean result = launcher.run(); + + LOG.info("Launcher run completed. Result=" + result); + Assert.assertTrue(result); + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/yarn-site.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/yarn-site.xml new file mode 100644 index 00000000000..7b02a822d9d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/resources/yarn-site.xml @@ -0,0 +1,21 @@ + + + + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index 881dbcbc057..1237a0d247d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -30,6 +30,7 @@ hadoop-yarn-applications-distributedshell + hadoop-yarn-applications-unmanaged-am-launcher From a2d560310631c85b9996f895e3a163c4f35e538e Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 24 Jul 2012 19:00:50 +0000 Subject: [PATCH 12/39] HADOOP-8606. FileSystem.get may return the wrong filesystem (Daryn Sharp via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365224 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../java/org/apache/hadoop/fs/FileSystem.java | 4 +- .../hadoop/fs/TestFileSystemCaching.java | 61 ++++++++++++++++++- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d4c4e6a07cb..ebd833bdb66 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -838,6 +838,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8599. Non empty response from FileSystem.getFileBlockLocations when asking for data beyond the end of file. (Andrey Klochkov via todd) + HADOOP-8606. FileSystem.get may return the wrong filesystem (Daryn Sharp + via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 62b48cf7467..f65d12c711c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -280,11 +280,11 @@ public abstract class FileSystem extends Configured implements Closeable { String scheme = uri.getScheme(); String authority = uri.getAuthority(); - if (scheme == null) { // no scheme: use default FS + if (scheme == null && authority == null) { // use default FS return get(conf); } - if (authority == null) { // no authority + if (scheme != null && authority == null) { // no authority URI defaultUri = getDefaultUri(conf); if (scheme.equals(defaultUri.getScheme()) // if scheme matches default && defaultUri.getAuthority() != null) { // & default has authority diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java index 1e176a0a12f..910139b56ec 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java @@ -34,8 +34,8 @@ import org.junit.Test; import java.security.PrivilegedExceptionAction; import java.util.concurrent.Semaphore; +import static org.junit.Assert.*; import static org.mockito.Mockito.mock; -import static junit.framework.Assert.assertTrue; public class TestFileSystemCaching { @@ -49,6 +49,65 @@ public class TestFileSystemCaching { assertSame(fs1, fs2); } + static class DefaultFs extends LocalFileSystem { + URI uri; + @Override + public void initialize(URI uri, Configuration conf) { + this.uri = uri; + } + @Override + public URI getUri() { + return uri; + } + } + + @Test + public void testDefaultFsUris() throws Exception { + final Configuration conf = new Configuration(); + conf.set("fs.defaultfs.impl", DefaultFs.class.getName()); + final URI defaultUri = URI.create("defaultfs://host"); + FileSystem.setDefaultUri(conf, defaultUri); + FileSystem fs = null; + + // sanity check default fs + final FileSystem defaultFs = FileSystem.get(conf); + assertEquals(defaultUri, defaultFs.getUri()); + + // has scheme, no auth + fs = FileSystem.get(URI.create("defaultfs:/"), conf); + assertSame(defaultFs, fs); + fs = FileSystem.get(URI.create("defaultfs:///"), conf); + assertSame(defaultFs, fs); + + // has scheme, same auth + fs = FileSystem.get(URI.create("defaultfs://host"), conf); + assertSame(defaultFs, fs); + + // has scheme, different auth + fs = FileSystem.get(URI.create("defaultfs://host2"), conf); + assertNotSame(defaultFs, fs); + + // no scheme, no auth + fs = FileSystem.get(URI.create("/"), conf); + assertSame(defaultFs, fs); + + // no scheme, same auth + try { + fs = FileSystem.get(URI.create("//host"), conf); + fail("got fs with auth but no scheme"); + } catch (Exception e) { + assertEquals("No FileSystem for scheme: null", e.getMessage()); + } + + // no scheme, different auth + try { + fs = FileSystem.get(URI.create("//host2"), conf); + fail("got fs with auth but no scheme"); + } catch (Exception e) { + assertEquals("No FileSystem for scheme: null", e.getMessage()); + } + } + public static class InitializeForeverFileSystem extends LocalFileSystem { final static Semaphore sem = new Semaphore(0); public void initialize(URI uri, Configuration conf) throws IOException { From cdef12644b949c711678f31d6cf3149d3b42b61b Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 24 Jul 2012 19:27:29 +0000 Subject: [PATCH 13/39] MAPREDUCE-4467. IndexCache failures due to missing synchronization (Kihwal Lee via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365240 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../org/apache/hadoop/mapred/IndexCache.java | 28 +++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 18863a1a4a0..0839a1486f6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -749,6 +749,9 @@ Release 0.23.3 - UNRELEASED maximum-am-resource-percent configurable on a per queue basis (tgraves via bobby) + MAPREDUCE-4467. IndexCache failures due to missing synchronization + (Kihwal Lee via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java index 2dbdf119602..54add3a81ac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java @@ -67,13 +67,13 @@ class IndexCache { if (info == null) { info = readIndexFileToCache(fileName, mapId, expectedIndexOwner); } else { - while (isUnderConstruction(info)) { - try { - // In case the entry is ready after the above check but - // before the following wait, we do timed wait. - info.wait(200); - } catch (InterruptedException e) { - throw new IOException("Interrupted waiting for construction", e); + synchronized(info) { + while (isUnderConstruction(info)) { + try { + info.wait(); + } catch (InterruptedException e) { + throw new IOException("Interrupted waiting for construction", e); + } } } LOG.debug("IndexCache HIT: MapId " + mapId + " found"); @@ -101,13 +101,13 @@ class IndexCache { IndexInformation info; IndexInformation newInd = new IndexInformation(); if ((info = cache.putIfAbsent(mapId, newInd)) != null) { - while (isUnderConstruction(info)) { - try { - // In case the entry is ready after the above check but - // before the following wait, we do timed wait. - info.wait(200); - } catch (InterruptedException e) { - throw new IOException("Interrupted waiting for construction", e); + synchronized(info) { + while (isUnderConstruction(info)) { + try { + info.wait(); + } catch (InterruptedException e) { + throw new IOException("Interrupted waiting for construction", e); + } } } LOG.debug("IndexCache HIT: MapId " + mapId + " found"); From 515a332912497792cbd862134a2eb86108a904ad Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 24 Jul 2012 22:08:27 +0000 Subject: [PATCH 14/39] MAPREDUCE-4465. Update description of yarn.nodemanager.address property. (bowang via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365322 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0839a1486f6..8bc35a5616d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -151,6 +151,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4407. Add hadoop-yarn-server-tests--tests.jar to hadoop dist package. (ahmed via tucu) + MAPREDUCE-4465. Update description of yarn.nodemanager.address property. + (bowang via tucu) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index ba570c3e5df..8aaaf38d4d7 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -241,7 +241,7 @@ - address of node manager IPC. + The address of the container manager in the NM. yarn.nodemanager.address 0.0.0.0:0 From 0bfa7d79d0fc5f127fd785e8ba3bc83dab8df991 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 25 Jul 2012 13:56:29 +0000 Subject: [PATCH 15/39] HADOOP-8551. fs -mkdir creates parent directories without the -p option (John George via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365588 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/shell/Mkdir.java | 8 +- .../src/test/resources/testHDFSConf.xml | 362 ++++++++++-------- 3 files changed, 209 insertions(+), 164 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ebd833bdb66..b5b671c7c15 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -841,6 +841,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8606. FileSystem.get may return the wrong filesystem (Daryn Sharp via bobby) + HADOOP-8551. fs -mkdir creates parent directories without the -p option + (John George via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java index eb0fd22562f..d92a5343c94 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Mkdir.java @@ -23,9 +23,11 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException; +import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; /** * Create the given dir @@ -66,7 +68,11 @@ class Mkdir extends FsCommand { @Override protected void processNonexistentPath(PathData item) throws IOException { - // TODO: should use createParents to control intermediate dir creation + // check if parent exists. this is complicated because getParent(a/b/c/) returns a/b/c, but + // we want a/b + if (!item.fs.exists(new Path(item.path.toString()).getParent()) && !createParents) { + throw new PathNotFoundException(item.toString()); + } if (!item.fs.mkdirs(item.path)) { throw new PathIOException(item.toString()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 1f6a6265ff3..a611c2e3c39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -129,7 +129,7 @@ ls: directory using relative path - -fs NAMENODE -mkdir dir1 + -fs NAMENODE -mkdir -p dir1 -fs NAMENODE -ls @@ -150,10 +150,10 @@ ls: directory using globbing - -fs NAMENODE -mkdir dir1 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir3 - -fs NAMENODE -mkdir dir4 + -fs NAMENODE -mkdir -p dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir3 + -fs NAMENODE -mkdir -p dir4 -fs NAMENODE -ls @@ -308,7 +308,7 @@ ls: Non-URI input dir at Namenode's path - -fs NAMENODE -mkdir NAMENODE/user/dir1 + -fs NAMENODE -mkdir -p NAMENODE/user/dir1 -fs NAMENODE -ls hdfs:///user/ @@ -329,7 +329,7 @@ ls: dir at hdfs:// path - -fs NAMENODE -mkdir hdfs:///user/dir1 + -fs NAMENODE -mkdir -p hdfs:///user/dir1 -fs NAMENODE -ls hdfs:///user/ @@ -350,9 +350,9 @@ ls: Non-URI input dir at Namenode's path using globing - -fs NAMENODE -mkdir NAMENODE/user/dir1 - -fs NAMENODE -mkdir NAMENODE/user/dir2 - -fs NAMENODE -mkdir NAMENODE/user/dir3 + -fs NAMENODE -mkdir -p NAMENODE/user/dir1 + -fs NAMENODE -mkdir -p NAMENODE/user/dir2 + -fs NAMENODE -mkdir -p NAMENODE/user/dir3 -fs NAMENODE -ls hdfs:///user/ @@ -381,9 +381,9 @@ ls: dir at hdfs:// path using globing - -fs NAMENODE -mkdir hdfs:///user/dir1 - -fs NAMENODE -mkdir hdfs:///user/dir2 - -fs NAMENODE -mkdir hdfs:///user/dir3 + -fs NAMENODE -mkdir -p hdfs:///user/dir1 + -fs NAMENODE -mkdir -p hdfs:///user/dir2 + -fs NAMENODE -mkdir -p hdfs:///user/dir3 -fs NAMENODE -ls hdfs:///user/ @@ -533,13 +533,13 @@ ls: files/directories using relative path - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 - -fs NAMENODE -mkdir dir0/dir1/dir1 - -fs NAMENODE -mkdir dir0/dir1/dir2 - -fs NAMENODE -mkdir dir0/dir2 - -fs NAMENODE -mkdir dir0/dir2/dir1 - -fs NAMENODE -mkdir dir0/dir2/dir2 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 + -fs NAMENODE -mkdir -p dir0/dir1/dir1 + -fs NAMENODE -mkdir -p dir0/dir1/dir2 + -fs NAMENODE -mkdir -p dir0/dir2 + -fs NAMENODE -mkdir -p dir0/dir2/dir1 + -fs NAMENODE -mkdir -p dir0/dir2/dir2 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -621,13 +621,13 @@ ls: files/directories using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 - -fs NAMENODE -mkdir dir0/dir1/dir1 - -fs NAMENODE -mkdir dir0/dir1/dir2 - -fs NAMENODE -mkdir dir0/dir2 - -fs NAMENODE -mkdir dir0/dir2/dir1 - -fs NAMENODE -mkdir dir0/dir2/dir2 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 + -fs NAMENODE -mkdir -p dir0/dir1/dir1 + -fs NAMENODE -mkdir -p dir0/dir1/dir2 + -fs NAMENODE -mkdir -p dir0/dir2 + -fs NAMENODE -mkdir -p dir0/dir2/dir1 + -fs NAMENODE -mkdir -p dir0/dir2/dir2 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -1045,7 +1045,7 @@ du: directory using relative path - -fs NAMENODE -mkdir dir0 + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -put CLITEST_DATA/data15bytes dir0/data15bytes -fs NAMENODE -du dir0 @@ -1352,13 +1352,13 @@ dus: directories/files using relative path - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 - -fs NAMENODE -mkdir dir0/dir1/dir1 - -fs NAMENODE -mkdir dir0/dir1/dir2 - -fs NAMENODE -mkdir dir0/dir2 - -fs NAMENODE -mkdir dir0/dir2/dir1 - -fs NAMENODE -mkdir dir0/dir2/dir2 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 + -fs NAMENODE -mkdir -p dir0/dir1/dir1 + -fs NAMENODE -mkdir -p dir0/dir1/dir2 + -fs NAMENODE -mkdir -p dir0/dir2 + -fs NAMENODE -mkdir -p dir0/dir2/dir1 + -fs NAMENODE -mkdir -p dir0/dir2/dir2 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -put CLITEST_DATA/data15bytes dir0/dir1/data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes dir0/dir1/data30bytes @@ -3275,7 +3275,7 @@ rm: removing a directory (relative path) - -fs NAMENODE mkdir dir0 + -fs NAMENODE mkdir -p dir0 -fs NAMENODE -rm dir0 @@ -3606,7 +3606,7 @@ rm: removing a directory (relative path) - -fs NAMENODE -mkdir dir0 + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -rm -r dir0 @@ -3655,10 +3655,10 @@ rm: removing directories by globbing (relative path) - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir1 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir3 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir3 -fs NAMENODE -rm -r dir* @@ -3986,7 +3986,7 @@ put: putting file into a directory(absolute path) - -fs NAMENODE -mkdir /dir0/dir1/data + -fs NAMENODE -mkdir -p /dir0/dir1/data -fs NAMENODE -put CLITEST_DATA/data15bytes /dir0/dir1/data --> -fs NAMENODE -du /dir0/dir1/data @@ -4004,7 +4004,7 @@ put: putting file into a directory(relative path) - -fs NAMENODE -mkdir dir0/dir1/data + -fs NAMENODE -mkdir -p dir0/dir1/data -fs NAMENODE -put CLITEST_DATA/data15bytes dir0/dir1/data -fs NAMENODE -du dir0/dir1/data @@ -4044,7 +4044,7 @@ put: putting many files into an existing directory(relative path) - -fs NAMENODE -mkdir dir0 + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -put CLITEST_DATA/data15bytes CLITEST_DATA/data30bytes dir0 -fs NAMENODE -du dir0 @@ -4206,7 +4206,7 @@ put: putting file into a directory in hdfs:// path - -fs NAMENODE -mkdir /dir1/data + -fs NAMENODE -mkdir -p /dir1/data -fs NAMENODE -put CLITEST_DATA/data15bytes hdfs:///dir1/data -fs NAMENODE -du hdfs:///dir1/data/ @@ -4329,7 +4329,7 @@ put: putting file into a directory in Namenode's path - -fs NAMENODE -mkdir /dir1/data + -fs NAMENODE -mkdir -p /dir1/data -fs NAMENODE -put CLITEST_DATA/data15bytes NAMENODE/dir1/data -fs NAMENODE -du NAMENODE/dir1/data @@ -4471,7 +4471,7 @@ copyFromLocal: copying file into a directory(absolute path) - -fs NAMENODE -mkdir /dir0/dir1/data + -fs NAMENODE -mkdir -p /dir0/dir1/data -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes /dir0/dir1/data -fs NAMENODE -du /dir0/dir1/data/* @@ -4489,7 +4489,7 @@ copyFromLocal: copying file into a directory(relative path) - -fs NAMENODE -mkdir dir0/dir1/data + -fs NAMENODE -mkdir -p dir0/dir1/data -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes dir0/dir1/data -fs NAMENODE -du dir0/dir1/data/* @@ -4529,7 +4529,7 @@ copyFromLocal: copying many files into an existing directory(relative path) - -fs NAMENODE -mkdir dir0 + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes CLITEST_DATA/data30bytes dir0 -fs NAMENODE -du dir0 @@ -5051,7 +5051,7 @@ cat: contents of files(relative path) using globbing - -fs NAMENODE -mkdir dir0 + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -put CLITEST_DATA/data15bytes dir0/data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes dir0/data30bytes -fs NAMENODE -put CLITEST_DATA/data60bytes dir0/data60bytes @@ -5093,7 +5093,7 @@ cat: contents of files(relative path) without globbing - -fs NAMENODE -mkdir dir0 + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -put CLITEST_DATA/data15bytes dir0/data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes dir0/data30bytes -fs NAMENODE -put CLITEST_DATA/data60bytes dir0/data60bytes @@ -5162,7 +5162,7 @@ cat: contents of directory(relative path) - -fs NAMENODE -mkdir dir1 + -fs NAMENODE -mkdir -p dir1 -fs NAMENODE -cat dir1 @@ -5444,11 +5444,30 @@ + + + mkdir: creating directory (absolute path) + + -fs NAMENODE -mkdir /dir0 + -fs NAMENODE -mkdir /dir0/b/ + -fs NAMENODE -du -s /dir0/b + + + -fs NAMENODE -rm /user + + + + RegexpComparator + ^0\s+/dir0/b + + + + mkdir: creating directory (relative path) - -fs NAMENODE -mkdir dir0 + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -du -s dir0 @@ -5497,10 +5516,10 @@ mkdir: creating many directories (relative path) - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir1 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir3 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir3 -fs NAMENODE -du -s dir* @@ -5547,7 +5566,7 @@ mkdir: creating a directory with the name of an already existing file -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes - -fs NAMENODE -mkdir data15bytes + -fs NAMENODE -mkdir -p data15bytes -fs NAMENODE -rm -r data15bytes @@ -5721,10 +5740,27 @@ - mkdir: Test recreate of existing directory fails + mkdir: Test create of directory with no parent and no -p fails -fs NAMENODE -rm -r -f dir0 -fs NAMENODE -mkdir dir0/dir1 + + + -fs NAMENODE -rm -r dir0 + + + + RegexpComparator + mkdir: `dir0/dir1': No such file or directory + + + + + + mkdir: Test recreate of existing directory fails + + -fs NAMENODE -rm -r -f dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -mkdir dir0/dir1 @@ -5742,7 +5778,7 @@ mkdir: Test recreate of existing directory with -p succeeds -fs NAMENODE -rm -r -f dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -mkdir -p dir0/dir1 @@ -6339,7 +6375,7 @@ stat: statistics about directory(relative path) - -fs NAMENODE -mkdir dirtest + -fs NAMENODE -mkdir -p dirtest -fs NAMENODE -stat "%n-%b-%o" dirtest @@ -6399,7 +6435,7 @@ -fs NAMENODE -put CLITEST_DATA/data30bytes data30bytes -fs NAMENODE -put CLITEST_DATA/data60bytes data60bytes -fs NAMENODE -put CLITEST_DATA/data120bytes data120bytes - -fs NAMENODE -mkdir datadir + -fs NAMENODE -mkdir -p datadir -fs NAMENODE -stat "%n-%b" data* @@ -6501,7 +6537,7 @@ -fs NAMENODE -put CLITEST_DATA/data30bytes hdfs:///dir0/data30bytes -fs NAMENODE -put CLITEST_DATA/data60bytes hdfs:///dir0/data60bytes -fs NAMENODE -put CLITEST_DATA/data120bytes hdfs:///dir0/data120bytes - -fs NAMENODE -mkdir hdfs:///dir0/datadir + -fs NAMENODE -mkdir -p hdfs:///dir0/datadir -fs NAMENODE -stat "%n-%b" hdfs:///dir0/data* @@ -6588,7 +6624,7 @@ -fs NAMENODE -put CLITEST_DATA/data30bytes NAMENODE/dir0/data30bytes -fs NAMENODE -put CLITEST_DATA/data60bytes NAMENODE/dir0/data60bytes -fs NAMENODE -put CLITEST_DATA/data120bytes NAMENODE/dir0/data120bytes - -fs NAMENODE -mkdir NAMENODE/dir0/datadir + -fs NAMENODE -mkdir -p NAMENODE/dir0/datadir -fs NAMENODE -stat "%n-%b" NAMENODE/dir0/data* @@ -6759,7 +6795,7 @@ tail: contents of directory(relative path) - -fs NAMENODE -mkdir dir1 + -fs NAMENODE -mkdir -p dir1 -fs NAMENODE -tail dir1 @@ -6966,7 +7002,7 @@ count: directory using relative path - -fs NAMENODE -mkdir dir1 + -fs NAMENODE -mkdir -p dir1 -fs NAMENODE -count dir1 @@ -7173,10 +7209,10 @@ count: relative path to multiple directories using globbing - -fs NAMENODE -mkdir dir1 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir3 - -fs NAMENODE -mkdir dir4 + -fs NAMENODE -mkdir -p dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir3 + -fs NAMENODE -mkdir -p dir4 -fs NAMENODE -count dir* @@ -7237,10 +7273,10 @@ count: relative path to multiple directories without globbing - -fs NAMENODE -mkdir dir1 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir3 - -fs NAMENODE -mkdir dir4 + -fs NAMENODE -mkdir -p dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir3 + -fs NAMENODE -mkdir -p dir4 -fs NAMENODE -count dir1 dir2 dir3 dir4 @@ -7322,7 +7358,7 @@ count: directory using relative path with -q option - -fs NAMENODE -mkdir dir1 + -fs NAMENODE -mkdir -p dir1 -fs NAMENODE -setQuota 10 dir1 -fs NAMENODE -setSpaceQuota 1m dir1 -fs NAMENODE -count -q dir1 @@ -7539,10 +7575,10 @@ count: relative path to multiple directories using globbing with -q option - -fs NAMENODE -mkdir dir1 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir3 - -fs NAMENODE -mkdir dir4 + -fs NAMENODE -mkdir -p dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir3 + -fs NAMENODE -mkdir -p dir4 -fs NAMENODE -setQuota 10 dir1 -fs NAMENODE -setSpaceQuota 1m dir1 -fs NAMENODE -setQuota 10 dir2 @@ -7619,10 +7655,10 @@ count: relative path to multiple directories without globbing with -q option - -fs NAMENODE -mkdir dir1 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir3 - -fs NAMENODE -mkdir dir4 + -fs NAMENODE -mkdir -p dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir3 + -fs NAMENODE -mkdir -p dir4 -fs NAMENODE -setQuota 10 dir1 -fs NAMENODE -setSpaceQuota 1m dir1 -fs NAMENODE -setQuota 10 dir2 @@ -8482,8 +8518,8 @@ chmod: change permission(octal mode) of directory in relative path - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -8594,8 +8630,8 @@ chmod: change permission(normal mode) of directory in relative path - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -8662,8 +8698,8 @@ chmod: change permission(octal mode) of directory in relative path recursively - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -8730,8 +8766,8 @@ chmod: change permission(normal mode) of directory in relative path recursively - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -8949,13 +8985,13 @@ chmod: change permission(octal mode) of multiple directories in relative path using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -9125,13 +9161,13 @@ chmod: change permission(octal mode) of multiple directories in relative path without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -9301,13 +9337,13 @@ chmod: change permission(normal mode) of multiple directories in relative path using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -9477,13 +9513,13 @@ chmod: change permission(normal mode) of multiple directories in relative path without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -9587,13 +9623,13 @@ chmod: change permission(octal mode) of multiple directories in relative path recursively using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -9697,13 +9733,13 @@ chmod: change permission(octal mode) of multiple directories in relative path recursively without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -9807,13 +9843,13 @@ chmod: change permission(normal mode) of multiple directories in relative path recursively using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -9917,13 +9953,13 @@ chmod: change permission(normal mode) of multiple directories in relative path recursively without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -12183,8 +12219,8 @@ chown: change ownership of directory in relative path - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -12251,8 +12287,8 @@ chown: change ownership of directory in relative path recursively - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -12438,13 +12474,13 @@ chown: change ownership of multiple directories in relative path using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -12614,13 +12650,13 @@ chown: change ownership of multiple directories in relative path without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -12724,13 +12760,13 @@ chown: change ownership of multiple directories recursively in relative path using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -12834,13 +12870,13 @@ chown: change ownership of multiple directories recursively in relative path without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -13462,8 +13498,8 @@ chgrp: change group of directory in relative path - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -13530,8 +13566,8 @@ chgrp: change group of directory in relative path recursively - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 @@ -13717,13 +13753,13 @@ chgrp: change group of multiple directories in relative path using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -13893,13 +13929,13 @@ chgrp: change group of multiple directories in relative path without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -14003,13 +14039,13 @@ chgrp: change group of multiple directories recursively in relative path using globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 @@ -14113,13 +14149,13 @@ chgrp: change group of multiple directories recursively in relative path without globbing - -fs NAMENODE -mkdir dir0 - -fs NAMENODE -mkdir dir0/dir1 + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -mkdir -p dir0/dir1 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/dir1/file1 -fs NAMENODE -touchz dir0/dir1/file2 - -fs NAMENODE -mkdir dir2 - -fs NAMENODE -mkdir dir2/dir1 + -fs NAMENODE -mkdir -p dir2 + -fs NAMENODE -mkdir -p dir2/dir1 -fs NAMENODE -touchz dir2/file0 -fs NAMENODE -touchz dir2/dir1/file1 -fs NAMENODE -touchz dir2/dir1/file2 From 56b0912b6cbbeff93dffbc92357b013c741fd850 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 25 Jul 2012 21:21:14 +0000 Subject: [PATCH 16/39] HDFS-3720. hdfs.h must get packaged. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365759 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/resources/assemblies/hadoop-dist.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml index 4762861e639..8db090062eb 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml @@ -111,9 +111,9 @@ /share/doc/hadoop/${hadoop.component} - ${basedir}/src/main/native + ${basedir}/src/main/native/libhdfs - *.h + hdfs.h /include diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9c002059ba0..2b1fb71d154 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -529,6 +529,8 @@ Branch-2 ( Unreleased changes ) HDFS-3709. TestStartup tests still binding to the ephemeral port. (eli) + HDFS-3720. hdfs.h must get packaged. (Colin Patrick McCabe via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) From 972953bd778081b9e8a0c3778d6df5c5e97368fa Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 25 Jul 2012 21:52:38 +0000 Subject: [PATCH 17/39] HDFS-3626. Creating file with invalid path can corrupt edit log. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365801 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/fs/FileUtil.java | 2 +- .../main/java/org/apache/hadoop/fs/Path.java | 2 +- .../java/org/apache/hadoop/fs/TestPath.java | 4 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 16 ++- .../hdfs/server/namenode/FSDirectory.java | 11 +- .../org/apache/hadoop/hdfs/TestDFSMkdirs.java | 61 +++++++--- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 8 ++ .../apache/hadoop/hdfs/TestFileCreation.java | 104 ++++++++++++++++++ 9 files changed, 185 insertions(+), 25 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index ba9bb4eafee..fd5166c69a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -611,7 +611,7 @@ public class FileUtil { */ public static int symLink(String target, String linkname) throws IOException{ String cmd = "ln -s " + target + " " + linkname; - Process p = Runtime.getRuntime().exec(cmd, null); + Process p = Runtime.getRuntime().exec(new String[]{"ln","-s",target,linkname}, null); int returnVal = -1; try{ returnVal = p.waitFor(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index 3d193dfad28..74c85af48bb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -139,7 +139,7 @@ public class Path implements Comparable { * Construct a path from a URI */ public Path(URI aUri) { - uri = aUri; + uri = aUri.normalize(); } /** Construct a Path from components. */ diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java index 2be0f9d26b6..5032caab911 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java @@ -61,7 +61,7 @@ public class TestPath extends TestCase { assertEquals(pathString, new Path(pathString).toString()); } - public void testNormalize() { + public void testNormalize() throws URISyntaxException { assertEquals("", new Path(".").toString()); assertEquals("..", new Path("..").toString()); assertEquals("/", new Path("/").toString()); @@ -75,6 +75,8 @@ public class TestPath extends TestCase { assertEquals("foo", new Path("foo/").toString()); assertEquals("foo", new Path("foo//").toString()); assertEquals("foo/bar", new Path("foo//bar").toString()); + assertEquals("hdfs://foo/foo2/bar/baz/", + new Path(new URI("hdfs://foo//foo2///bar/baz///")).toString()); if (Path.WINDOWS) { assertEquals("c:/a/b", new Path("c:\\a\\b").toString()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2b1fb71d154..e955ea34ecc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -531,6 +531,8 @@ Branch-2 ( Unreleased changes ) HDFS-3720. hdfs.h must get packaged. (Colin Patrick McCabe via atm) + HDFS-3626. Creating file with invalid path can corrupt edit log (todd) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index a6adc81532b..0246620e905 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -56,6 +56,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -118,7 +119,7 @@ public class DFSUtil { /** * Whether the pathname is valid. Currently prohibits relative paths, - * and names which contain a ":" or "/" + * names which contain a ":" or "//", or other non-canonical paths. */ public static boolean isValidName(String src) { // Path must be absolute. @@ -127,15 +128,22 @@ public class DFSUtil { } // Check for ".." "." ":" "/" - StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR); - while(tokens.hasMoreTokens()) { - String element = tokens.nextToken(); + String[] components = StringUtils.split(src, '/'); + for (int i = 0; i < components.length; i++) { + String element = components[i]; if (element.equals("..") || element.equals(".") || (element.indexOf(":") >= 0) || (element.indexOf("/") >= 0)) { return false; } + + // The string may start or end with a /, but not have + // "//" in the middle. + if (element.isEmpty() && i != components.length - 1 && + i != 0) { + return false; + } } return true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index d7aabedc56a..fb9f54d21b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -230,8 +230,15 @@ public class FSDirectory implements Closeable { // Always do an implicit mkdirs for parent directory tree. long modTime = now(); - if (!mkdirs(new Path(path).getParent().toString(), permissions, true, - modTime)) { + + Path parent = new Path(path).getParent(); + if (parent == null) { + // Trying to add "/" as a file - this path has no + // parent -- avoids an NPE below. + return null; + } + + if (!mkdirs(parent.toString(), permissions, true, modTime)) { return null; } INodeFileUnderConstruction newNode = new INodeFileUnderConstruction( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java index 71f0c130bf1..bea29f9c67d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java @@ -17,8 +17,7 @@ */ package org.apache.hadoop.hdfs; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.io.DataOutputStream; import java.io.FileNotFoundException; @@ -26,33 +25,38 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.junit.Test; - /** - * This class tests that the DFS command mkdirs cannot create subdirectories - * from a file when passed an illegal path. HADOOP-281. + * This class tests that the DFS command mkdirs only creates valid + * directories, and generally behaves as expected. */ public class TestDFSMkdirs { + private Configuration conf = new HdfsConfiguration(); + + private static final String[] NON_CANONICAL_PATHS = new String[] { + "//test1", + "/test2/..", + "/test2//bar", + "/test2/../test4", + "/test5/." + }; - private void writeFile(FileSystem fileSys, Path name) throws IOException { - DataOutputStream stm = fileSys.create(name); - stm.writeBytes("wchien"); - stm.close(); - } - /** * Tests mkdirs can create a directory that does not exist and will - * not create a subdirectory off a file. + * not create a subdirectory off a file. Regression test for HADOOP-281. */ @Test public void testDFSMkdirs() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); FileSystem fileSys = cluster.getFileSystem(); try { // First create a new directory with mkdirs @@ -63,7 +67,7 @@ public class TestDFSMkdirs { // Second, create a file in that directory. Path myFile = new Path("/test/mkdirs/myFile"); - writeFile(fileSys, myFile); + DFSTestUtil.writeFile(fileSys, myFile, "hello world"); // Third, use mkdir to create a subdirectory off of that file, // and check that it fails. @@ -99,7 +103,7 @@ public class TestDFSMkdirs { // Create a dir when parent dir exists as a file, should fail IOException expectedException = null; String filePath = "/mkdir-file-" + Time.now(); - writeFile(dfs, new Path(filePath)); + DFSTestUtil.writeFile(dfs, new Path(filePath), "hello world"); try { dfs.mkdir(new Path(filePath + "/mkdir"), FsPermission.getDefault()); } catch (IOException e) { @@ -126,4 +130,29 @@ public class TestDFSMkdirs { cluster.shutdown(); } } + + /** + * Regression test for HDFS-3626. Creates a file using a non-canonical path + * (i.e. with extra slashes between components) and makes sure that the NN + * rejects it. + */ + @Test + public void testMkdirRpcNonCanonicalPath() throws IOException { + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + try { + NamenodeProtocols nnrpc = cluster.getNameNodeRpc(); + + for (String pathStr : NON_CANONICAL_PATHS) { + try { + nnrpc.mkdirs(pathStr, new FsPermission((short)0755), true); + fail("Did not fail when called with a non-canonicalized path: " + + pathStr); + } catch (InvalidPathException ipe) { + // expected + } + } + } finally { + cluster.shutdown(); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index af1f6d6ea57..06b4ed0cb8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -620,4 +620,12 @@ public class TestDFSUtil { assertEquals(1, uris.size()); assertTrue(uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR))); } + + @Test + public void testIsValidName() { + assertFalse(DFSUtil.isValidName("/foo/../bar")); + assertFalse(DFSUtil.isValidName("/foo//bar")); + assertTrue(DFSUtil.isValidName("/")); + assertTrue(DFSUtil.isValidName("/bar/")); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index c3909e38139..cb79f67c905 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -41,6 +41,8 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; import java.net.UnknownHostException; import java.util.EnumSet; @@ -53,6 +55,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -69,7 +72,10 @@ import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.junit.Ignore; @@ -93,6 +99,15 @@ public class TestFileCreation { static final int numBlocks = 2; static final int fileSize = numBlocks * blockSize + 1; boolean simulatedStorage = false; + + private static final String[] NON_CANONICAL_PATHS = new String[] { + "//foo", + "///foo2", + "//dir//file", + "////test2/file", + "/dir/./file2", + "/dir/../file3" + }; // creates a file but does not close it public static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl) @@ -988,4 +1003,93 @@ public class TestFileCreation { } } } + + /** + * Regression test for HDFS-3626. Creates a file using a non-canonical path + * (i.e. with extra slashes between components) and makes sure that the NN + * can properly restart. + * + * This test RPCs directly to the NN, to ensure that even an old client + * which passes an invalid path won't cause corrupt edits. + */ + @Test + public void testCreateNonCanonicalPathAndRestartRpc() throws Exception { + doCreateTest(CreationMethod.DIRECT_NN_RPC); + } + + /** + * Another regression test for HDFS-3626. This one creates files using + * a Path instantiated from a string object. + */ + @Test + public void testCreateNonCanonicalPathAndRestartFromString() + throws Exception { + doCreateTest(CreationMethod.PATH_FROM_STRING); + } + + /** + * Another regression test for HDFS-3626. This one creates files using + * a Path instantiated from a URI object. + */ + @Test + public void testCreateNonCanonicalPathAndRestartFromUri() + throws Exception { + doCreateTest(CreationMethod.PATH_FROM_URI); + } + + private static enum CreationMethod { + DIRECT_NN_RPC, + PATH_FROM_URI, + PATH_FROM_STRING + }; + private void doCreateTest(CreationMethod method) throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1).build(); + try { + FileSystem fs = cluster.getFileSystem(); + NamenodeProtocols nnrpc = cluster.getNameNodeRpc(); + + for (String pathStr : NON_CANONICAL_PATHS) { + System.out.println("Creating " + pathStr + " by " + method); + switch (method) { + case DIRECT_NN_RPC: + try { + nnrpc.create(pathStr, new FsPermission((short)0755), "client", + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), + true, (short)1, 128*1024*1024L); + fail("Should have thrown exception when creating '" + + pathStr + "'" + " by " + method); + } catch (InvalidPathException ipe) { + // When we create by direct NN RPC, the NN just rejects the + // non-canonical paths, rather than trying to normalize them. + // So, we expect all of them to fail. + } + break; + + case PATH_FROM_URI: + case PATH_FROM_STRING: + // Unlike the above direct-to-NN case, we expect these to succeed, + // since the Path constructor should normalize the path. + Path p; + if (method == CreationMethod.PATH_FROM_URI) { + p = new Path(new URI(fs.getUri() + pathStr)); + } else { + p = new Path(fs.getUri() + pathStr); + } + FSDataOutputStream stm = fs.create(p); + IOUtils.closeStream(stm); + break; + default: + throw new AssertionError("bad method: " + method); + } + } + + cluster.restartNameNode(); + + } finally { + cluster.shutdown(); + } + } + } From 06b394dcf90d38a724ef9b8779a507d0b196d358 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 25 Jul 2012 22:04:59 +0000 Subject: [PATCH 18/39] Amend previous commit of HDFS-3626: accidentally included a hunk from HADOOP-8621 in svn commit. Reverting that hunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365817 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/java/org/apache/hadoop/fs/FileUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index fd5166c69a6..ba9bb4eafee 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -611,7 +611,7 @@ public class FileUtil { */ public static int symLink(String target, String linkname) throws IOException{ String cmd = "ln -s " + target + " " + linkname; - Process p = Runtime.getRuntime().exec(new String[]{"ln","-s",target,linkname}, null); + Process p = Runtime.getRuntime().exec(cmd, null); int returnVal = -1; try{ returnVal = p.waitFor(); From 8f395c2f78e5813e613197c3078a4ebc5d42775a Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 25 Jul 2012 23:37:25 +0000 Subject: [PATCH 19/39] HDFS-3696. Set chunked streaming mode in WebHdfsFileSystem write operations to get around a Java library bug causing OutOfMemoryError. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365839 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 18 +- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 199 ++++++++++++++++++ 3 files changed, 206 insertions(+), 14 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e955ea34ecc..5d1e01ef65d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1403,6 +1403,9 @@ Release 0.23.3 - UNRELEASED HDFS-3646. LeaseRenewer can hold reference to inactive DFSClient instances forever. (Kihwal Lee via daryn) + HDFS-3696. Set chunked streaming mode in WebHdfsFileSystem write operations + to get around a Java library bug causing OutOfMemoryError. (szetszwo) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 4327dd18c5b..420d74d5829 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -423,6 +423,7 @@ public class WebHdfsFileSystem extends FileSystem //Step 2) Submit another Http request with the URL from the Location header with data. conn = (HttpURLConnection)new URL(redirect).openConnection(); conn.setRequestMethod(op.getType().toString()); + conn.setChunkedStreamingMode(32 << 10); //32kB-chunk return conn; } @@ -835,8 +836,7 @@ public class WebHdfsFileSystem extends FileSystem } private static WebHdfsFileSystem getWebHdfs( - final Token token, final Configuration conf - ) throws IOException, InterruptedException, URISyntaxException { + final Token token, final Configuration conf) throws IOException { final InetSocketAddress nnAddr = SecurityUtil.getTokenServiceAddr(token); final URI uri = DFSUtil.createUri(WebHdfsFileSystem.SCHEME, nnAddr); @@ -850,12 +850,7 @@ public class WebHdfsFileSystem extends FileSystem // update the kerberos credentials, if they are coming from a keytab ugi.reloginFromKeytab(); - try { - WebHdfsFileSystem webhdfs = getWebHdfs(token, conf); - return webhdfs.renewDelegationToken(token); - } catch (URISyntaxException e) { - throw new IOException(e); - } + return getWebHdfs(token, conf).renewDelegationToken(token); } @Override @@ -865,12 +860,7 @@ public class WebHdfsFileSystem extends FileSystem // update the kerberos credentials, if they are coming from a keytab ugi.checkTGTAndReloginFromKeytab(); - try { - final WebHdfsFileSystem webhdfs = getWebHdfs(token, conf); - webhdfs.cancelDelegationToken(token); - } catch (URISyntaxException e) { - throw new IOException(e); - } + getWebHdfs(token, conf).cancelDelegationToken(token); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java new file mode 100644 index 00000000000..66f65cc2792 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.web; + +import java.io.IOException; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.Assert; +import org.junit.Test; + +/** Test WebHDFS */ +public class TestWebHDFS { + static final Log LOG = LogFactory.getLog(TestWebHDFS.class); + + static final Random RANDOM = new Random(); + + static final long systemStartTime = System.nanoTime(); + + /** A timer for measuring performance. */ + static class Ticker { + final String name; + final long startTime = System.nanoTime(); + private long previousTick = startTime; + + Ticker(final String name, String format, Object... args) { + this.name = name; + LOG.info(String.format("\n\n%s START: %s\n", + name, String.format(format, args))); + } + + void tick(final long nBytes, String format, Object... args) { + final long now = System.nanoTime(); + if (now - previousTick > 10000000000L) { + previousTick = now; + final double mintues = (now - systemStartTime)/60000000000.0; + LOG.info(String.format("\n\n%s %.2f min) %s %s\n", name, mintues, + String.format(format, args), toMpsString(nBytes, now))); + } + } + + void end(final long nBytes) { + final long now = System.nanoTime(); + final double seconds = (now - startTime)/1000000000.0; + LOG.info(String.format("\n\n%s END: duration=%.2fs %s\n", + name, seconds, toMpsString(nBytes, now))); + } + + String toMpsString(final long nBytes, final long now) { + final double mb = nBytes/(double)(1<<20); + final double mps = mb*1000000000.0/(now - startTime); + return String.format("[nBytes=%.2fMB, speed=%.2fMB/s]", mb, mps); + } + } + + @Test + public void testLargeFile() throws Exception { + largeFileTest(200L << 20); //200MB file length + } + + /** Test read and write large files. */ + static void largeFileTest(final long fileLength) throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(3) + .build(); + try { + cluster.waitActive(); + + final FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf); + final Path dir = new Path("/test/largeFile"); + Assert.assertTrue(fs.mkdirs(dir)); + + final byte[] data = new byte[1 << 20]; + RANDOM.nextBytes(data); + + final byte[] expected = new byte[2 * data.length]; + System.arraycopy(data, 0, expected, 0, data.length); + System.arraycopy(data, 0, expected, data.length, data.length); + + final Path p = new Path(dir, "file"); + final Ticker t = new Ticker("WRITE", "fileLength=" + fileLength); + final FSDataOutputStream out = fs.create(p); + try { + long remaining = fileLength; + for(; remaining > 0;) { + t.tick(fileLength - remaining, "remaining=%d", remaining); + + final int n = (int)Math.min(remaining, data.length); + out.write(data, 0, n); + remaining -= n; + } + } finally { + out.close(); + } + t.end(fileLength); + + Assert.assertEquals(fileLength, fs.getFileStatus(p).getLen()); + + final long smallOffset = RANDOM.nextInt(1 << 20) + (1 << 20); + final long largeOffset = fileLength - smallOffset; + final byte[] buf = new byte[data.length]; + + verifySeek(fs, p, largeOffset, fileLength, buf, expected); + verifySeek(fs, p, smallOffset, fileLength, buf, expected); + + verifyPread(fs, p, largeOffset, fileLength, buf, expected); + } finally { + cluster.shutdown(); + } + } + + static void checkData(long offset, long remaining, int n, + byte[] actual, byte[] expected) { + if (RANDOM.nextInt(100) == 0) { + int j = (int)(offset % actual.length); + for(int i = 0; i < n; i++) { + if (expected[j] != actual[i]) { + Assert.fail("expected[" + j + "]=" + expected[j] + + " != actual[" + i + "]=" + actual[i] + + ", offset=" + offset + ", remaining=" + remaining + ", n=" + n); + } + j++; + } + } + } + + /** test seek */ + static void verifySeek(FileSystem fs, Path p, long offset, long length, + byte[] buf, byte[] expected) throws IOException { + long remaining = length - offset; + long checked = 0; + LOG.info("XXX SEEK: offset=" + offset + ", remaining=" + remaining); + + final Ticker t = new Ticker("SEEK", "offset=%d, remaining=%d", + offset, remaining); + final FSDataInputStream in = fs.open(p, 64 << 10); + in.seek(offset); + for(; remaining > 0; ) { + t.tick(checked, "offset=%d, remaining=%d", offset, remaining); + final int n = (int)Math.min(remaining, buf.length); + in.readFully(buf, 0, n); + checkData(offset, remaining, n, buf, expected); + + offset += n; + remaining -= n; + checked += n; + } + in.close(); + t.end(checked); + } + + static void verifyPread(FileSystem fs, Path p, long offset, long length, + byte[] buf, byte[] expected) throws IOException { + long remaining = length - offset; + long checked = 0; + LOG.info("XXX PREAD: offset=" + offset + ", remaining=" + remaining); + + final Ticker t = new Ticker("PREAD", "offset=%d, remaining=%d", + offset, remaining); + final FSDataInputStream in = fs.open(p, 64 << 10); + for(; remaining > 0; ) { + t.tick(checked, "offset=%d, remaining=%d", offset, remaining); + final int n = (int)Math.min(remaining, buf.length); + in.readFully(offset, buf, 0, n); + checkData(offset, remaining, n, buf, expected); + + offset += n; + remaining -= n; + checked += n; + } + in.close(); + t.end(checked); + } +} From 9d16c9354b0c05edb30d23003dcdec4cc44ed925 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 26 Jul 2012 13:23:05 +0000 Subject: [PATCH 20/39] MAPREDUCE-4417. add support for encrypted shuffle (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365979 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/conf/ssl-client.xml.example | 27 +- .../src/main/conf/ssl-server.xml.example | 26 +- .../ssl/FileBasedKeyStoresFactory.java | 241 ++++++++ .../hadoop/security/ssl/KeyStoresFactory.java | 67 ++ .../ssl/ReloadingX509TrustManager.java | 204 ++++++ .../hadoop/security/ssl/SSLFactory.java | 237 +++++++ .../security/ssl/SSLHostnameVerifier.java | 585 ++++++++++++++++++ .../src/main/resources/core-default.xml | 47 ++ .../hadoop/security/ssl/KeyStoreTestUtil.java | 270 ++++++++ .../ssl/TestReloadingX509TrustManager.java | 175 ++++++ .../hadoop/security/ssl/TestSSLFactory.java | 164 +++++ hadoop-mapreduce-project/CHANGES.txt | 2 + .../dev-support/findbugs-exclude.xml | 7 +- .../mapreduce/v2/app/job/impl/TaskImpl.java | 14 +- .../org/apache/hadoop/mapreduce/MRConfig.java | 5 + .../hadoop/mapreduce/task/reduce/Fetcher.java | 40 +- .../src/main/resources/mapred-default.xml | 15 + .../security/ssl/TestEncryptedShuffle.java | 184 ++++++ .../apache/hadoop/mapred/ShuffleHandler.java | 87 ++- .../src/site/apt/EncryptedShuffle.apt.vm | 320 ++++++++++ .../src/site/apt/index.apt.vm | 2 + hadoop-project/src/site/site.xml | 1 + 22 files changed, 2688 insertions(+), 32 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/KeyStoresFactory.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm diff --git a/hadoop-common-project/hadoop-common/src/main/conf/ssl-client.xml.example b/hadoop-common-project/hadoop-common/src/main/conf/ssl-client.xml.example index ec3fd41fa81..a50dce48c23 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/ssl-client.xml.example +++ b/hadoop-common-project/hadoop-common/src/main/conf/ssl-client.xml.example @@ -1,6 +1,21 @@ + @@ -21,7 +36,15 @@ ssl.client.truststore.type jks - Optional. Default value is "jks". + Optional. The keystore file format, default value is "jks". + + + + + ssl.client.truststore.reload.interval + 10000 + Truststore reload check interval, in milliseconds. + Default value is 10000 (10 seconds). @@ -50,7 +73,7 @@ ssl.client.keystore.type jks - Optional. Default value is "jks". + Optional. The keystore file format, default value is "jks". diff --git a/hadoop-common-project/hadoop-common/src/main/conf/ssl-server.xml.example b/hadoop-common-project/hadoop-common/src/main/conf/ssl-server.xml.example index 22e9cb0ebbf..4b363ff21f7 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/ssl-server.xml.example +++ b/hadoop-common-project/hadoop-common/src/main/conf/ssl-server.xml.example @@ -1,6 +1,21 @@ + @@ -20,10 +35,17 @@ ssl.server.truststore.type jks - Optional. Default value is "jks". + Optional. The keystore file format, default value is "jks". + + ssl.server.truststore.reload.interval + 10000 + Truststore reload check interval, in milliseconds. + Default value is 10000 (10 seconds). + + ssl.server.keystore.location @@ -48,7 +70,7 @@ ssl.server.keystore.type jks - Optional. Default value is "jks". + Optional. The keystore file format, default value is "jks". diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java new file mode 100644 index 00000000000..00dd2021eec --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java @@ -0,0 +1,241 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.security.ssl; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManager; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.text.MessageFormat; + +/** + * {@link KeyStoresFactory} implementation that reads the certificates from + * keystore files. + *

+ * if the trust certificates keystore file changes, the {@link TrustManager} + * is refreshed with the new trust certificate entries (using a + * {@link ReloadingX509TrustManager} trustmanager). + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class FileBasedKeyStoresFactory implements KeyStoresFactory { + + private static final Log LOG = + LogFactory.getLog(FileBasedKeyStoresFactory.class); + + public static final String SSL_KEYSTORE_LOCATION_TPL_KEY = + "ssl.{0}.keystore.location"; + public static final String SSL_KEYSTORE_PASSWORD_TPL_KEY = + "ssl.{0}.keystore.password"; + public static final String SSL_KEYSTORE_TYPE_TPL_KEY = + "ssl.{0}.keystore.type"; + + public static final String SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY = + "ssl.{0}.truststore.reload.interval"; + public static final String SSL_TRUSTSTORE_LOCATION_TPL_KEY = + "ssl.{0}.truststore.location"; + public static final String SSL_TRUSTSTORE_PASSWORD_TPL_KEY = + "ssl.{0}.truststore.password"; + public static final String SSL_TRUSTSTORE_TYPE_TPL_KEY = + "ssl.{0}.truststore.type"; + + /** + * Default format of the keystore files. + */ + public static final String DEFAULT_KEYSTORE_TYPE = "jks"; + + /** + * Reload interval in milliseconds. + */ + public static final int DEFAULT_SSL_TRUSTSTORE_RELOAD_INTERVAL = 10000; + + private Configuration conf; + private KeyManager[] keyManagers; + private TrustManager[] trustManagers; + private ReloadingX509TrustManager trustManager; + + /** + * Resolves a property name to its client/server version if applicable. + *

+ * NOTE: This method is public for testing purposes. + * + * @param mode client/server mode. + * @param template property name template. + * @return the resolved property name. + */ + @VisibleForTesting + public static String resolvePropertyName(SSLFactory.Mode mode, + String template) { + return MessageFormat.format(template, mode.toString().toLowerCase()); + } + + /** + * Sets the configuration for the factory. + * + * @param conf the configuration for the factory. + */ + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + /** + * Returns the configuration of the factory. + * + * @return the configuration of the factory. + */ + @Override + public Configuration getConf() { + return conf; + } + + /** + * Initializes the keystores of the factory. + * + * @param mode if the keystores are to be used in client or server mode. + * @throws IOException thrown if the keystores could not be initialized due + * to an IO error. + * @throws GeneralSecurityException thrown if the keystores could not be + * initialized due to a security error. + */ + public void init(SSLFactory.Mode mode) + throws IOException, GeneralSecurityException { + + boolean requireClientCert = + conf.getBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, true); + + // certificate store + String keystoreType = + conf.get(resolvePropertyName(mode, SSL_KEYSTORE_TYPE_TPL_KEY), + DEFAULT_KEYSTORE_TYPE); + KeyStore keystore = KeyStore.getInstance(keystoreType); + String keystorePassword = null; + if (requireClientCert || mode == SSLFactory.Mode.SERVER) { + String locationProperty = + resolvePropertyName(mode, SSL_KEYSTORE_LOCATION_TPL_KEY); + String keystoreLocation = conf.get(locationProperty, ""); + if (keystoreLocation.isEmpty()) { + throw new GeneralSecurityException("The property '" + locationProperty + + "' has not been set in the ssl configuration file."); + } + String passwordProperty = + resolvePropertyName(mode, SSL_KEYSTORE_PASSWORD_TPL_KEY); + keystorePassword = conf.get(passwordProperty, ""); + if (keystorePassword.isEmpty()) { + throw new GeneralSecurityException("The property '" + passwordProperty + + "' has not been set in the ssl configuration file."); + } + LOG.debug(mode.toString() + " KeyStore: " + keystoreLocation); + + InputStream is = new FileInputStream(keystoreLocation); + try { + keystore.load(is, keystorePassword.toCharArray()); + } finally { + is.close(); + } + LOG.info(mode.toString() + " Loaded KeyStore: " + keystoreLocation); + } else { + keystore.load(null, null); + } + KeyManagerFactory keyMgrFactory = KeyManagerFactory.getInstance("SunX509"); + keyMgrFactory.init(keystore, (keystorePassword != null) ? + keystorePassword.toCharArray() : null); + keyManagers = keyMgrFactory.getKeyManagers(); + + //trust store + String truststoreType = + conf.get(resolvePropertyName(mode, SSL_TRUSTSTORE_TYPE_TPL_KEY), + DEFAULT_KEYSTORE_TYPE); + + String locationProperty = + resolvePropertyName(mode, SSL_TRUSTSTORE_LOCATION_TPL_KEY); + String truststoreLocation = conf.get(locationProperty, ""); + if (truststoreLocation.isEmpty()) { + throw new GeneralSecurityException("The property '" + locationProperty + + "' has not been set in the ssl configuration file."); + } + + String passwordProperty = resolvePropertyName(mode, + SSL_TRUSTSTORE_PASSWORD_TPL_KEY); + String truststorePassword = conf.get(passwordProperty, ""); + if (truststorePassword.isEmpty()) { + throw new GeneralSecurityException("The property '" + passwordProperty + + "' has not been set in the ssl configuration file."); + } + long truststoreReloadInterval = + conf.getLong( + resolvePropertyName(mode, SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), + DEFAULT_SSL_TRUSTSTORE_RELOAD_INTERVAL); + + LOG.debug(mode.toString() + " TrustStore: " + truststoreLocation); + + trustManager = new ReloadingX509TrustManager(truststoreType, + truststoreLocation, + truststorePassword, + truststoreReloadInterval); + trustManager.init(); + LOG.info(mode.toString() + " Loaded TrustStore: " + truststoreLocation); + + trustManagers = new TrustManager[]{trustManager}; + } + + /** + * Releases any resources being used. + */ + @Override + public synchronized void destroy() { + if (trustManager != null) { + trustManager.destroy(); + trustManager = null; + keyManagers = null; + trustManagers = null; + } + } + + /** + * Returns the keymanagers for owned certificates. + * + * @return the keymanagers for owned certificates. + */ + @Override + public KeyManager[] getKeyManagers() { + return keyManagers; + } + + /** + * Returns the trustmanagers for trusted certificates. + * + * @return the trustmanagers for trusted certificates. + */ + @Override + public TrustManager[] getTrustManagers() { + return trustManagers; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/KeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/KeyStoresFactory.java new file mode 100644 index 00000000000..25d2c543362 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/KeyStoresFactory.java @@ -0,0 +1,67 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.security.ssl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configurable; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.TrustManager; +import java.io.IOException; +import java.security.GeneralSecurityException; + +/** + * Interface that gives access to {@link KeyManager} and {@link TrustManager} + * implementations. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface KeyStoresFactory extends Configurable { + + /** + * Initializes the keystores of the factory. + * + * @param mode if the keystores are to be used in client or server mode. + * @throws IOException thrown if the keystores could not be initialized due + * to an IO error. + * @throws GeneralSecurityException thrown if the keystores could not be + * initialized due to an security error. + */ + public void init(SSLFactory.Mode mode) throws IOException, GeneralSecurityException; + + /** + * Releases any resources being used. + */ + public void destroy(); + + /** + * Returns the keymanagers for owned certificates. + * + * @return the keymanagers for owned certificates. + */ + public KeyManager[] getKeyManagers(); + + /** + * Returns the trustmanagers for trusted certificates. + * + * @return the trustmanagers for trusted certificates. + */ + public TrustManager[] getTrustManagers(); + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java new file mode 100644 index 00000000000..58cdf00175f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.ssl; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A {@link TrustManager} implementation that reloads its configuration when + * the truststore file on disk changes. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class ReloadingX509TrustManager + implements X509TrustManager, Runnable { + + private static final Log LOG = + LogFactory.getLog(ReloadingX509TrustManager.class); + + private String type; + private File file; + private String password; + private long lastLoaded; + private long reloadInterval; + private AtomicReference trustManagerRef; + + private volatile boolean running; + private Thread reloader; + + /** + * Creates a reloadable trustmanager. The trustmanager reloads itself + * if the underlying trustore file has changed. + * + * @param type type of truststore file, typically 'jks'. + * @param location local path to the truststore file. + * @param password password of the truststore file. + * @param reloadInterval interval to check if the truststore file has + * changed, in milliseconds. + * @throws IOException thrown if the truststore could not be initialized due + * to an IO error. + * @throws GeneralSecurityException thrown if the truststore could not be + * initialized due to a security error. + */ + public ReloadingX509TrustManager(String type, String location, + String password, long reloadInterval) + throws IOException, GeneralSecurityException { + this.type = type; + file = new File(location); + this.password = password; + trustManagerRef = new AtomicReference(); + trustManagerRef.set(loadTrustManager()); + this.reloadInterval = reloadInterval; + } + + /** + * Starts the reloader thread. + */ + public void init() { + reloader = new Thread(this, "Truststore reloader thread"); + reloader.setDaemon(true); + running = true; + reloader.start(); + } + + /** + * Stops the reloader thread. + */ + public void destroy() { + running = false; + reloader.interrupt(); + } + + /** + * Returns the reload check interval. + * + * @return the reload check interval, in milliseconds. + */ + public long getReloadInterval() { + return reloadInterval; + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) + throws CertificateException { + X509TrustManager tm = trustManagerRef.get(); + if (tm != null) { + tm.checkClientTrusted(chain, authType); + } else { + throw new CertificateException("Unknown client chain certificate: " + + chain[0].toString()); + } + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) + throws CertificateException { + X509TrustManager tm = trustManagerRef.get(); + if (tm != null) { + tm.checkServerTrusted(chain, authType); + } else { + throw new CertificateException("Unknown server chain certificate: " + + chain[0].toString()); + } + } + + private static final X509Certificate[] EMPTY = new X509Certificate[0]; + @Override + public X509Certificate[] getAcceptedIssuers() { + X509Certificate[] issuers = EMPTY; + X509TrustManager tm = trustManagerRef.get(); + if (tm != null) { + issuers = tm.getAcceptedIssuers(); + } + return issuers; + } + + boolean needsReload() { + boolean reload = true; + if (file.exists()) { + if (file.lastModified() == lastLoaded) { + reload = false; + } + } else { + lastLoaded = 0; + } + return reload; + } + + X509TrustManager loadTrustManager() + throws IOException, GeneralSecurityException { + X509TrustManager trustManager = null; + KeyStore ks = KeyStore.getInstance(type); + lastLoaded = file.lastModified(); + FileInputStream in = new FileInputStream(file); + try { + ks.load(in, password.toCharArray()); + LOG.debug("Loaded truststore '" + file + "'"); + } finally { + in.close(); + } + + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance("SunX509"); + trustManagerFactory.init(ks); + TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); + for (TrustManager trustManager1 : trustManagers) { + if (trustManager1 instanceof X509TrustManager) { + trustManager = (X509TrustManager) trustManager1; + break; + } + } + return trustManager; + } + + @Override + public void run() { + while (running) { + try { + Thread.sleep(reloadInterval); + } catch (InterruptedException e) { + //NOP + } + if (running && needsReload()) { + try { + trustManagerRef.set(loadTrustManager()); + } catch (Exception ex) { + LOG.warn("Could not load truststore (keep using existing one) : " + + ex.toString(), ex); + } + } + } + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java new file mode 100644 index 00000000000..adab8b6395a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java @@ -0,0 +1,237 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.security.ssl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLServerSocketFactory; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; +import java.security.GeneralSecurityException; + +/** + * Factory that creates SSLEngine and SSLSocketFactory instances using + * Hadoop configuration information. + *

+ * This SSLFactory uses a {@link ReloadingX509TrustManager} instance, + * which reloads public keys if the truststore file changes. + *

+ * This factory is used to configure HTTPS in Hadoop HTTP based endpoints, both + * client and server. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class SSLFactory { + + @InterfaceAudience.Private + public static enum Mode { CLIENT, SERVER } + + public static final String SSL_REQUIRE_CLIENT_CERT_KEY = + "hadoop.ssl.require.client.cert"; + public static final String SSL_HOSTNAME_VERIFIER_KEY = + "hadoop.ssl.hostname.verifier"; + public static final String SSL_CLIENT_CONF_KEY = + "hadoop.ssl.client.conf"; + public static final String SSL_SERVER_CONF_KEY = + "hadoop.ssl.server.conf"; + + public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT = false; + + public static final String KEYSTORES_FACTORY_CLASS_KEY = + "hadoop.ssl.keystores.factory.class"; + + private Configuration conf; + private Mode mode; + private boolean requireClientCert; + private SSLContext context; + private HostnameVerifier hostnameVerifier; + private KeyStoresFactory keystoresFactory; + + /** + * Creates an SSLFactory. + * + * @param mode SSLFactory mode, client or server. + * @param conf Hadoop configuration from where the SSLFactory configuration + * will be read. + */ + public SSLFactory(Mode mode, Configuration conf) { + this.conf = conf; + if (mode == null) { + throw new IllegalArgumentException("mode cannot be NULL"); + } + this.mode = mode; + requireClientCert = conf.getBoolean(SSL_REQUIRE_CLIENT_CERT_KEY, + DEFAULT_SSL_REQUIRE_CLIENT_CERT); + Configuration sslConf = readSSLConfiguration(mode); + + Class klass + = conf.getClass(KEYSTORES_FACTORY_CLASS_KEY, + FileBasedKeyStoresFactory.class, KeyStoresFactory.class); + keystoresFactory = ReflectionUtils.newInstance(klass, sslConf); + } + + private Configuration readSSLConfiguration(Mode mode) { + Configuration sslConf = new Configuration(false); + sslConf.setBoolean(SSL_REQUIRE_CLIENT_CERT_KEY, requireClientCert); + String sslConfResource; + if (mode == Mode.CLIENT) { + sslConfResource = conf.get(SSL_CLIENT_CONF_KEY, "ssl-client.xml"); + } else { + sslConfResource = conf.get(SSL_SERVER_CONF_KEY, "ssl-server.xml"); + } + sslConf.addResource(sslConfResource); + return sslConf; + } + + /** + * Initializes the factory. + * + * @throws GeneralSecurityException thrown if an SSL initialization error + * happened. + * @throws IOException thrown if an IO error happened while reading the SSL + * configuration. + */ + public void init() throws GeneralSecurityException, IOException { + keystoresFactory.init(mode); + context = SSLContext.getInstance("TLS"); + context.init(keystoresFactory.getKeyManagers(), + keystoresFactory.getTrustManagers(), null); + + hostnameVerifier = getHostnameVerifier(conf); + } + + private HostnameVerifier getHostnameVerifier(Configuration conf) + throws GeneralSecurityException, IOException { + HostnameVerifier hostnameVerifier; + String verifier = + conf.get(SSL_HOSTNAME_VERIFIER_KEY, "DEFAULT").trim().toUpperCase(); + if (verifier.equals("DEFAULT")) { + hostnameVerifier = SSLHostnameVerifier.DEFAULT; + } else if (verifier.equals("DEFAULT_AND_LOCALHOST")) { + hostnameVerifier = SSLHostnameVerifier.DEFAULT_AND_LOCALHOST; + } else if (verifier.equals("STRICT")) { + hostnameVerifier = SSLHostnameVerifier.STRICT; + } else if (verifier.equals("STRICT_IE6")) { + hostnameVerifier = SSLHostnameVerifier.STRICT_IE6; + } else if (verifier.equals("ALLOW_ALL")) { + hostnameVerifier = SSLHostnameVerifier.ALLOW_ALL; + } else { + throw new GeneralSecurityException("Invalid hostname verifier: " + + verifier); + } + return hostnameVerifier; + } + + /** + * Releases any resources being used. + */ + public void destroy() { + keystoresFactory.destroy(); + } + /** + * Returns the SSLFactory KeyStoresFactory instance. + * + * @return the SSLFactory KeyStoresFactory instance. + */ + public KeyStoresFactory getKeystoresFactory() { + return keystoresFactory; + } + + /** + * Returns a configured SSLEngine. + * + * @return the configured SSLEngine. + * @throws GeneralSecurityException thrown if the SSL engine could not + * be initialized. + * @throws IOException thrown if and IO error occurred while loading + * the server keystore. + */ + public SSLEngine createSSLEngine() + throws GeneralSecurityException, IOException { + SSLEngine sslEngine = context.createSSLEngine(); + if (mode == Mode.CLIENT) { + sslEngine.setUseClientMode(true); + } else { + sslEngine.setUseClientMode(false); + sslEngine.setNeedClientAuth(requireClientCert); + } + return sslEngine; + } + + /** + * Returns a configured SSLServerSocketFactory. + * + * @return the configured SSLSocketFactory. + * @throws GeneralSecurityException thrown if the SSLSocketFactory could not + * be initialized. + * @throws IOException thrown if and IO error occurred while loading + * the server keystore. + */ + public SSLServerSocketFactory createSSLServerSocketFactory() + throws GeneralSecurityException, IOException { + if (mode != Mode.SERVER) { + throw new IllegalStateException("Factory is in CLIENT mode"); + } + return context.getServerSocketFactory(); + } + + /** + * Returns a configured SSLSocketFactory. + * + * @return the configured SSLSocketFactory. + * @throws GeneralSecurityException thrown if the SSLSocketFactory could not + * be initialized. + * @throws IOException thrown if and IO error occurred while loading + * the server keystore. + */ + public SSLSocketFactory createSSLSocketFactory() + throws GeneralSecurityException, IOException { + if (mode != Mode.CLIENT) { + throw new IllegalStateException("Factory is in CLIENT mode"); + } + return context.getSocketFactory(); + } + + /** + * Returns the hostname verifier it should be used in HttpsURLConnections. + * + * @return the hostname verifier. + */ + public HostnameVerifier getHostnameVerifier() { + if (mode != Mode.CLIENT) { + throw new IllegalStateException("Factory is in CLIENT mode"); + } + return hostnameVerifier; + } + + /** + * Returns if client certificates are required or not. + * + * @return if client certificates are required or not. + */ + public boolean isClientCertRequired() { + return requireClientCert; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java new file mode 100644 index 00000000000..3f88fb89a7d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java @@ -0,0 +1,585 @@ +/* + * $HeadURL$ + * $Revision$ + * $Date$ + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.hadoop.security.ssl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.io.IOException; +import java.io.InputStream; +import java.security.cert.Certificate; +import java.security.cert.CertificateParsingException; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.StringTokenizer; +import java.util.TreeSet; + +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocket; + +/** + ************************************************************************ + * Copied from the not-yet-commons-ssl project at + * http://juliusdavies.ca/commons-ssl/ + * This project is not yet in Apache, but it is Apache 2.0 licensed. + ************************************************************************ + * Interface for checking if a hostname matches the names stored inside the + * server's X.509 certificate. Correctly implements + * javax.net.ssl.HostnameVerifier, but that interface is not recommended. + * Instead we added several check() methods that take SSLSocket, + * or X509Certificate, or ultimately (they all end up calling this one), + * String. (It's easier to supply JUnit with Strings instead of mock + * SSLSession objects!) + *

Our check() methods throw exceptions if the name is + * invalid, whereas javax.net.ssl.HostnameVerifier just returns true/false. + *

+ * We provide the HostnameVerifier.DEFAULT, HostnameVerifier.STRICT, and + * HostnameVerifier.ALLOW_ALL implementations. We also provide the more + * specialized HostnameVerifier.DEFAULT_AND_LOCALHOST, as well as + * HostnameVerifier.STRICT_IE6. But feel free to define your own + * implementations! + *

+ * Inspired by Sebastian Hauer's original StrictSSLProtocolSocketFactory in the + * HttpClient "contrib" repository. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface SSLHostnameVerifier extends javax.net.ssl.HostnameVerifier { + + boolean verify(String host, SSLSession session); + + void check(String host, SSLSocket ssl) throws IOException; + + void check(String host, X509Certificate cert) throws SSLException; + + void check(String host, String[] cns, String[] subjectAlts) + throws SSLException; + + void check(String[] hosts, SSLSocket ssl) throws IOException; + + void check(String[] hosts, X509Certificate cert) throws SSLException; + + + /** + * Checks to see if the supplied hostname matches any of the supplied CNs + * or "DNS" Subject-Alts. Most implementations only look at the first CN, + * and ignore any additional CNs. Most implementations do look at all of + * the "DNS" Subject-Alts. The CNs or Subject-Alts may contain wildcards + * according to RFC 2818. + * + * @param cns CN fields, in order, as extracted from the X.509 + * certificate. + * @param subjectAlts Subject-Alt fields of type 2 ("DNS"), as extracted + * from the X.509 certificate. + * @param hosts The array of hostnames to verify. + * @throws SSLException If verification failed. + */ + void check(String[] hosts, String[] cns, String[] subjectAlts) + throws SSLException; + + + /** + * The DEFAULT HostnameVerifier works the same way as Curl and Firefox. + *

+ * The hostname must match either the first CN, or any of the subject-alts. + * A wildcard can occur in the CN, and in any of the subject-alts. + *

+ * The only difference between DEFAULT and STRICT is that a wildcard (such + * as "*.foo.com") with DEFAULT matches all subdomains, including + * "a.b.foo.com". + */ + public final static SSLHostnameVerifier DEFAULT = + new AbstractVerifier() { + public final void check(final String[] hosts, final String[] cns, + final String[] subjectAlts) + throws SSLException { + check(hosts, cns, subjectAlts, false, false); + } + + public final String toString() { return "DEFAULT"; } + }; + + + /** + * The DEFAULT_AND_LOCALHOST HostnameVerifier works like the DEFAULT + * one with one additional relaxation: a host of "localhost", + * "localhost.localdomain", "127.0.0.1", "::1" will always pass, no matter + * what is in the server's certificate. + */ + public final static SSLHostnameVerifier DEFAULT_AND_LOCALHOST = + new AbstractVerifier() { + public final void check(final String[] hosts, final String[] cns, + final String[] subjectAlts) + throws SSLException { + if (isLocalhost(hosts[0])) { + return; + } + check(hosts, cns, subjectAlts, false, false); + } + + public final String toString() { return "DEFAULT_AND_LOCALHOST"; } + }; + + /** + * The STRICT HostnameVerifier works the same way as java.net.URL in Sun + * Java 1.4, Sun Java 5, Sun Java 6. It's also pretty close to IE6. + * This implementation appears to be compliant with RFC 2818 for dealing + * with wildcards. + *

+ * The hostname must match either the first CN, or any of the subject-alts. + * A wildcard can occur in the CN, and in any of the subject-alts. The + * one divergence from IE6 is how we only check the first CN. IE6 allows + * a match against any of the CNs present. We decided to follow in + * Sun Java 1.4's footsteps and only check the first CN. + *

+ * A wildcard such as "*.foo.com" matches only subdomains in the same + * level, for example "a.foo.com". It does not match deeper subdomains + * such as "a.b.foo.com". + */ + public final static SSLHostnameVerifier STRICT = + new AbstractVerifier() { + public final void check(final String[] host, final String[] cns, + final String[] subjectAlts) + throws SSLException { + check(host, cns, subjectAlts, false, true); + } + + public final String toString() { return "STRICT"; } + }; + + /** + * The STRICT_IE6 HostnameVerifier works just like the STRICT one with one + * minor variation: the hostname can match against any of the CN's in the + * server's certificate, not just the first one. This behaviour is + * identical to IE6's behaviour. + */ + public final static SSLHostnameVerifier STRICT_IE6 = + new AbstractVerifier() { + public final void check(final String[] host, final String[] cns, + final String[] subjectAlts) + throws SSLException { + check(host, cns, subjectAlts, true, true); + } + + public final String toString() { return "STRICT_IE6"; } + }; + + /** + * The ALLOW_ALL HostnameVerifier essentially turns hostname verification + * off. This implementation is a no-op, and never throws the SSLException. + */ + public final static SSLHostnameVerifier ALLOW_ALL = + new AbstractVerifier() { + public final void check(final String[] host, final String[] cns, + final String[] subjectAlts) { + // Allow everything - so never blowup. + } + + public final String toString() { return "ALLOW_ALL"; } + }; + + @SuppressWarnings("unchecked") + abstract class AbstractVerifier implements SSLHostnameVerifier { + + /** + * This contains a list of 2nd-level domains that aren't allowed to + * have wildcards when combined with country-codes. + * For example: [*.co.uk]. + *

+ * The [*.co.uk] problem is an interesting one. Should we just hope + * that CA's would never foolishly allow such a certificate to happen? + * Looks like we're the only implementation guarding against this. + * Firefox, Curl, Sun Java 1.4, 5, 6 don't bother with this check. + */ + private final static String[] BAD_COUNTRY_2LDS = + {"ac", "co", "com", "ed", "edu", "go", "gouv", "gov", "info", + "lg", "ne", "net", "or", "org"}; + + private final static String[] LOCALHOSTS = {"::1", "127.0.0.1", + "localhost", + "localhost.localdomain"}; + + + static { + // Just in case developer forgot to manually sort the array. :-) + Arrays.sort(BAD_COUNTRY_2LDS); + Arrays.sort(LOCALHOSTS); + } + + protected AbstractVerifier() {} + + /** + * The javax.net.ssl.HostnameVerifier contract. + * + * @param host 'hostname' we used to create our socket + * @param session SSLSession with the remote server + * @return true if the host matched the one in the certificate. + */ + public boolean verify(String host, SSLSession session) { + try { + Certificate[] certs = session.getPeerCertificates(); + X509Certificate x509 = (X509Certificate) certs[0]; + check(new String[]{host}, x509); + return true; + } + catch (SSLException e) { + return false; + } + } + + public void check(String host, SSLSocket ssl) throws IOException { + check(new String[]{host}, ssl); + } + + public void check(String host, X509Certificate cert) + throws SSLException { + check(new String[]{host}, cert); + } + + public void check(String host, String[] cns, String[] subjectAlts) + throws SSLException { + check(new String[]{host}, cns, subjectAlts); + } + + public void check(String host[], SSLSocket ssl) + throws IOException { + if (host == null) { + throw new NullPointerException("host to verify is null"); + } + + SSLSession session = ssl.getSession(); + if (session == null) { + // In our experience this only happens under IBM 1.4.x when + // spurious (unrelated) certificates show up in the server' + // chain. Hopefully this will unearth the real problem: + InputStream in = ssl.getInputStream(); + in.available(); + /* + If you're looking at the 2 lines of code above because + you're running into a problem, you probably have two + options: + + #1. Clean up the certificate chain that your server + is presenting (e.g. edit "/etc/apache2/server.crt" + or wherever it is your server's certificate chain + is defined). + + OR + + #2. Upgrade to an IBM 1.5.x or greater JVM, or switch + to a non-IBM JVM. + */ + + // If ssl.getInputStream().available() didn't cause an + // exception, maybe at least now the session is available? + session = ssl.getSession(); + if (session == null) { + // If it's still null, probably a startHandshake() will + // unearth the real problem. + ssl.startHandshake(); + + // Okay, if we still haven't managed to cause an exception, + // might as well go for the NPE. Or maybe we're okay now? + session = ssl.getSession(); + } + } + Certificate[] certs; + try { + certs = session.getPeerCertificates(); + } catch (SSLPeerUnverifiedException spue) { + InputStream in = ssl.getInputStream(); + in.available(); + // Didn't trigger anything interesting? Okay, just throw + // original. + throw spue; + } + X509Certificate x509 = (X509Certificate) certs[0]; + check(host, x509); + } + + public void check(String[] host, X509Certificate cert) + throws SSLException { + String[] cns = Certificates.getCNs(cert); + String[] subjectAlts = Certificates.getDNSSubjectAlts(cert); + check(host, cns, subjectAlts); + } + + public void check(final String[] hosts, final String[] cns, + final String[] subjectAlts, final boolean ie6, + final boolean strictWithSubDomains) + throws SSLException { + // Build up lists of allowed hosts For logging/debugging purposes. + StringBuffer buf = new StringBuffer(32); + buf.append('<'); + for (int i = 0; i < hosts.length; i++) { + String h = hosts[i]; + h = h != null ? h.trim().toLowerCase() : ""; + hosts[i] = h; + if (i > 0) { + buf.append('/'); + } + buf.append(h); + } + buf.append('>'); + String hostnames = buf.toString(); + // Build the list of names we're going to check. Our DEFAULT and + // STRICT implementations of the HostnameVerifier only use the + // first CN provided. All other CNs are ignored. + // (Firefox, wget, curl, Sun Java 1.4, 5, 6 all work this way). + TreeSet names = new TreeSet(); + if (cns != null && cns.length > 0 && cns[0] != null) { + names.add(cns[0]); + if (ie6) { + for (int i = 1; i < cns.length; i++) { + names.add(cns[i]); + } + } + } + if (subjectAlts != null) { + for (int i = 0; i < subjectAlts.length; i++) { + if (subjectAlts[i] != null) { + names.add(subjectAlts[i]); + } + } + } + if (names.isEmpty()) { + String msg = "Certificate for " + hosts[0] + " doesn't contain CN or DNS subjectAlt"; + throw new SSLException(msg); + } + + // StringBuffer for building the error message. + buf = new StringBuffer(); + + boolean match = false; + out: + for (Iterator it = names.iterator(); it.hasNext();) { + // Don't trim the CN, though! + String cn = (String) it.next(); + cn = cn.toLowerCase(); + // Store CN in StringBuffer in case we need to report an error. + buf.append(" <"); + buf.append(cn); + buf.append('>'); + if (it.hasNext()) { + buf.append(" OR"); + } + + // The CN better have at least two dots if it wants wildcard + // action. It also can't be [*.co.uk] or [*.co.jp] or + // [*.org.uk], etc... + boolean doWildcard = cn.startsWith("*.") && + cn.lastIndexOf('.') >= 0 && + !isIP4Address(cn) && + acceptableCountryWildcard(cn); + + for (int i = 0; i < hosts.length; i++) { + final String hostName = hosts[i].trim().toLowerCase(); + if (doWildcard) { + match = hostName.endsWith(cn.substring(1)); + if (match && strictWithSubDomains) { + // If we're in strict mode, then [*.foo.com] is not + // allowed to match [a.b.foo.com] + match = countDots(hostName) == countDots(cn); + } + } else { + match = hostName.equals(cn); + } + if (match) { + break out; + } + } + } + if (!match) { + throw new SSLException("hostname in certificate didn't match: " + hostnames + " !=" + buf); + } + } + + public static boolean isIP4Address(final String cn) { + boolean isIP4 = true; + String tld = cn; + int x = cn.lastIndexOf('.'); + // We only bother analyzing the characters after the final dot + // in the name. + if (x >= 0 && x + 1 < cn.length()) { + tld = cn.substring(x + 1); + } + for (int i = 0; i < tld.length(); i++) { + if (!Character.isDigit(tld.charAt(0))) { + isIP4 = false; + break; + } + } + return isIP4; + } + + public static boolean acceptableCountryWildcard(final String cn) { + int cnLen = cn.length(); + if (cnLen >= 7 && cnLen <= 9) { + // Look for the '.' in the 3rd-last position: + if (cn.charAt(cnLen - 3) == '.') { + // Trim off the [*.] and the [.XX]. + String s = cn.substring(2, cnLen - 3); + // And test against the sorted array of bad 2lds: + int x = Arrays.binarySearch(BAD_COUNTRY_2LDS, s); + return x < 0; + } + } + return true; + } + + public static boolean isLocalhost(String host) { + host = host != null ? host.trim().toLowerCase() : ""; + if (host.startsWith("::1")) { + int x = host.lastIndexOf('%'); + if (x >= 0) { + host = host.substring(0, x); + } + } + int x = Arrays.binarySearch(LOCALHOSTS, host); + return x >= 0; + } + + /** + * Counts the number of dots "." in a string. + * + * @param s string to count dots from + * @return number of dots + */ + public static int countDots(final String s) { + int count = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == '.') { + count++; + } + } + return count; + } + } + + @SuppressWarnings("unchecked") + static class Certificates { + public static String[] getCNs(X509Certificate cert) { + LinkedList cnList = new LinkedList(); + /* + Sebastian Hauer's original StrictSSLProtocolSocketFactory used + getName() and had the following comment: + + Parses a X.500 distinguished name for the value of the + "Common Name" field. This is done a bit sloppy right + now and should probably be done a bit more according to + RFC 2253. + + I've noticed that toString() seems to do a better job than + getName() on these X500Principal objects, so I'm hoping that + addresses Sebastian's concern. + + For example, getName() gives me this: + 1.2.840.113549.1.9.1=#16166a756c6975736461766965734063756362632e636f6d + + whereas toString() gives me this: + EMAILADDRESS=juliusdavies@cucbc.com + + Looks like toString() even works with non-ascii domain names! + I tested it with "花子.co.jp" and it worked fine. + */ + String subjectPrincipal = cert.getSubjectX500Principal().toString(); + StringTokenizer st = new StringTokenizer(subjectPrincipal, ","); + while (st.hasMoreTokens()) { + String tok = st.nextToken(); + int x = tok.indexOf("CN="); + if (x >= 0) { + cnList.add(tok.substring(x + 3)); + } + } + if (!cnList.isEmpty()) { + String[] cns = new String[cnList.size()]; + cnList.toArray(cns); + return cns; + } else { + return null; + } + } + + + /** + * Extracts the array of SubjectAlt DNS names from an X509Certificate. + * Returns null if there aren't any. + *

+ * Note: Java doesn't appear able to extract international characters + * from the SubjectAlts. It can only extract international characters + * from the CN field. + *

+ * (Or maybe the version of OpenSSL I'm using to test isn't storing the + * international characters correctly in the SubjectAlts?). + * + * @param cert X509Certificate + * @return Array of SubjectALT DNS names stored in the certificate. + */ + public static String[] getDNSSubjectAlts(X509Certificate cert) { + LinkedList subjectAltList = new LinkedList(); + Collection c = null; + try { + c = cert.getSubjectAlternativeNames(); + } + catch (CertificateParsingException cpe) { + // Should probably log.debug() this? + cpe.printStackTrace(); + } + if (c != null) { + Iterator it = c.iterator(); + while (it.hasNext()) { + List list = (List) it.next(); + int type = ((Integer) list.get(0)).intValue(); + // If type is 2, then we've got a dNSName + if (type == 2) { + String s = (String) list.get(1); + subjectAltList.add(s); + } + } + } + if (!subjectAltList.isEmpty()) { + String[] subjectAlts = new String[subjectAltList.size()]; + subjectAltList.toArray(subjectAlts); + return subjectAlts; + } else { + return null; + } + } + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index ce28a2b2942..7667eecb244 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1026,4 +1026,51 @@ hadoop.http.staticuser.user dr.who + + + + + hadoop.ssl.keystores.factory.class + org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory + + The keystores factory to use for retrieving certificates. + + + + + hadoop.ssl.require.client.cert + false + Whether client certificates are required + + + + hadoop.ssl.hostname.verifier + DEFAULT + + The hostname verifier to provide for HttpsURLConnections. + Valid values are: DEFAULT, STRICT, STRICT_I6, DEFAULT_AND_LOCALHOST and + ALLOW_ALL + + + + + hadoop.ssl.server.conf + ssl-server.xml + + Resource file from which ssl server keystore information will be extracted. + This file is looked up in the classpath, typically it should be in Hadoop + conf/ directory. + + + + + hadoop.ssl.client.conf + ssl-client.xml + + Resource file from which ssl client keystore information will be extracted + This file is looked up in the classpath, typically it should be in Hadoop + conf/ directory. + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java new file mode 100644 index 00000000000..c57cbfdd96e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java @@ -0,0 +1,270 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.ssl; + +import org.apache.hadoop.conf.Configuration; +import sun.security.x509.AlgorithmId; +import sun.security.x509.CertificateAlgorithmId; +import sun.security.x509.CertificateIssuerName; +import sun.security.x509.CertificateSerialNumber; +import sun.security.x509.CertificateSubjectName; +import sun.security.x509.CertificateValidity; +import sun.security.x509.CertificateVersion; +import sun.security.x509.CertificateX509Key; +import sun.security.x509.X500Name; +import sun.security.x509.X509CertImpl; +import sun.security.x509.X509CertInfo; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.math.BigInteger; +import java.net.URL; +import java.security.GeneralSecurityException; +import java.security.Key; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +public class KeyStoreTestUtil { + + public static String getClasspathDir(Class klass) throws Exception { + String file = klass.getName(); + file = file.replace('.', '/') + ".class"; + URL url = Thread.currentThread().getContextClassLoader().getResource(file); + String baseDir = url.toURI().getPath(); + baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1); + return baseDir; + } + + /** + * Create a self-signed X.509 Certificate. + * From http://bfo.com/blog/2011/03/08/odds_and_ends_creating_a_new_x_509_certificate.html. + * + * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB" + * @param pair the KeyPair + * @param days how many days from now the Certificate is valid for + * @param algorithm the signing algorithm, eg "SHA1withRSA" + * @return the self-signed certificate + * @throws IOException thrown if an IO error ocurred. + * @throws GeneralSecurityException thrown if an Security error ocurred. + */ + public static X509Certificate generateCertificate(String dn, KeyPair pair, + int days, String algorithm) + throws GeneralSecurityException, IOException { + PrivateKey privkey = pair.getPrivate(); + X509CertInfo info = new X509CertInfo(); + Date from = new Date(); + Date to = new Date(from.getTime() + days * 86400000l); + CertificateValidity interval = new CertificateValidity(from, to); + BigInteger sn = new BigInteger(64, new SecureRandom()); + X500Name owner = new X500Name(dn); + + info.set(X509CertInfo.VALIDITY, interval); + info.set(X509CertInfo.SERIAL_NUMBER, new CertificateSerialNumber(sn)); + info.set(X509CertInfo.SUBJECT, new CertificateSubjectName(owner)); + info.set(X509CertInfo.ISSUER, new CertificateIssuerName(owner)); + info.set(X509CertInfo.KEY, new CertificateX509Key(pair.getPublic())); + info + .set(X509CertInfo.VERSION, new CertificateVersion(CertificateVersion.V3)); + AlgorithmId algo = new AlgorithmId(AlgorithmId.md5WithRSAEncryption_oid); + info.set(X509CertInfo.ALGORITHM_ID, new CertificateAlgorithmId(algo)); + + // Sign the cert to identify the algorithm that's used. + X509CertImpl cert = new X509CertImpl(info); + cert.sign(privkey, algorithm); + + // Update the algorith, and resign. + algo = (AlgorithmId) cert.get(X509CertImpl.SIG_ALG); + info + .set(CertificateAlgorithmId.NAME + "." + CertificateAlgorithmId.ALGORITHM, + algo); + cert = new X509CertImpl(info); + cert.sign(privkey, algorithm); + return cert; + } + + public static KeyPair generateKeyPair(String algorithm) + throws NoSuchAlgorithmException { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm); + keyGen.initialize(1024); + return keyGen.genKeyPair(); + } + + private static KeyStore createEmptyKeyStore() + throws GeneralSecurityException, IOException { + KeyStore ks = KeyStore.getInstance("JKS"); + ks.load(null, null); // initialize + return ks; + } + + private static void saveKeyStore(KeyStore ks, String filename, + String password) + throws GeneralSecurityException, IOException { + FileOutputStream out = new FileOutputStream(filename); + try { + ks.store(out, password.toCharArray()); + } finally { + out.close(); + } + } + + public static void createKeyStore(String filename, + String password, String alias, + Key privateKey, Certificate cert) + throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + ks.setKeyEntry(alias, privateKey, password.toCharArray(), + new Certificate[]{cert}); + saveKeyStore(ks, filename, password); + } + + public static void createTrustStore(String filename, + String password, String alias, + Certificate cert) + throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + ks.setCertificateEntry(alias, cert); + saveKeyStore(ks, filename, password); + } + + public static void createTrustStore( + String filename, String password, Map certs) + throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + for (Map.Entry cert : certs.entrySet()) { + ks.setCertificateEntry(cert.getKey(), cert.getValue()); + } + saveKeyStore(ks, filename, password); + } + + public static void cleanupSSLConfig(String keystoresDir, String sslConfDir) + throws Exception { + File f = new File(keystoresDir + "/clientKS.jks"); + f.delete(); + f = new File(keystoresDir + "/serverKS.jks"); + f.delete(); + f = new File(keystoresDir + "/trustKS.jks"); + f.delete(); + f = new File(sslConfDir + "/ssl-client.xml"); + f.delete(); + f = new File(sslConfDir + "/ssl-server.xml"); + f.delete(); + } + + public static void setupSSLConfig(String keystoresDir, String sslConfDir, + Configuration conf, boolean useClientCert) + throws Exception { + String clientKS = keystoresDir + "/clientKS.jks"; + String clientPassword = "clientP"; + String serverKS = keystoresDir + "/serverKS.jks"; + String serverPassword = "serverP"; + String trustKS = keystoresDir + "/trustKS.jks"; + String trustPassword = "trustP"; + + File sslClientConfFile = new File(sslConfDir + "/ssl-client.xml"); + File sslServerConfFile = new File(sslConfDir + "/ssl-server.xml"); + + Map certs = new HashMap(); + + if (useClientCert) { + KeyPair cKP = KeyStoreTestUtil.generateKeyPair("RSA"); + X509Certificate cCert = + KeyStoreTestUtil.generateCertificate("CN=localhost, O=client", cKP, 30, + "SHA1withRSA"); + KeyStoreTestUtil.createKeyStore(clientKS, clientPassword, "client", + cKP.getPrivate(), cCert); + certs.put("client", cCert); + } + + KeyPair sKP = KeyStoreTestUtil.generateKeyPair("RSA"); + X509Certificate sCert = + KeyStoreTestUtil.generateCertificate("CN=localhost, O=server", sKP, 30, + "SHA1withRSA"); + KeyStoreTestUtil.createKeyStore(serverKS, serverPassword, "server", + sKP.getPrivate(), sCert); + certs.put("server", sCert); + + KeyStoreTestUtil.createTrustStore(trustKS, trustPassword, certs); + + Configuration clientSSLConf = new Configuration(false); + clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.CLIENT, + FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), clientKS); + clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.CLIENT, + FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), clientPassword); + clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.CLIENT, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS); + clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.CLIENT, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), trustPassword); + clientSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.CLIENT, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000"); + + Configuration serverSSLConf = new Configuration(false); + serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), serverKS); + serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), serverPassword); + serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS); + serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), trustPassword); + serverSSLConf.set(FileBasedKeyStoresFactory.resolvePropertyName( + SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000"); + + Writer writer = new FileWriter(sslClientConfFile); + try { + clientSSLConf.writeXml(writer); + } finally { + writer.close(); + } + + writer = new FileWriter(sslServerConfFile); + try { + serverSSLConf.writeXml(writer); + } finally { + writer.close(); + } + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL"); + conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName()); + conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName()); + conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java new file mode 100644 index 00000000000..5a03605f349 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security.ssl; + +import org.apache.hadoop.fs.FileUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.util.HashMap; +import java.util.Map; + +import static junit.framework.Assert.assertEquals; +import static org.apache.hadoop.security.ssl.KeyStoreTestUtil.createTrustStore; +import static org.apache.hadoop.security.ssl.KeyStoreTestUtil.generateCertificate; +import static org.apache.hadoop.security.ssl.KeyStoreTestUtil.generateKeyPair; + +public class TestReloadingX509TrustManager { + + private static final String BASEDIR = + System.getProperty("test.build.data", "target/test-dir") + "/" + + TestReloadingX509TrustManager.class.getSimpleName(); + + private X509Certificate cert1; + private X509Certificate cert2; + + @BeforeClass + public static void setUp() throws Exception { + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + } + + @Test(expected = IOException.class) + public void testLoadMissingTrustStore() throws Exception { + String truststoreLocation = BASEDIR + "/testmissing.jks"; + + ReloadingX509TrustManager tm = + new ReloadingX509TrustManager("jks", truststoreLocation, "password", 10); + try { + tm.init(); + } finally { + tm.destroy(); + } + } + + @Test(expected = IOException.class) + public void testLoadCorruptTrustStore() throws Exception { + String truststoreLocation = BASEDIR + "/testcorrupt.jks"; + OutputStream os = new FileOutputStream(truststoreLocation); + os.write(1); + os.close(); + + ReloadingX509TrustManager tm = + new ReloadingX509TrustManager("jks", truststoreLocation, "password", 10); + try { + tm.init(); + } finally { + tm.destroy(); + } + } + + @Test + public void testReload() throws Exception { + KeyPair kp = generateKeyPair("RSA"); + cert1 = generateCertificate("CN=Cert1", kp, 30, "SHA1withRSA"); + cert2 = generateCertificate("CN=Cert2", kp, 30, "SHA1withRSA"); + String truststoreLocation = BASEDIR + "/testreload.jks"; + createTrustStore(truststoreLocation, "password", "cert1", cert1); + + ReloadingX509TrustManager tm = + new ReloadingX509TrustManager("jks", truststoreLocation, "password", 10); + try { + tm.init(); + assertEquals(1, tm.getAcceptedIssuers().length); + + // Wait so that the file modification time is different + Thread.sleep((tm.getReloadInterval() + 1000)); + + // Add another cert + Map certs = new HashMap(); + certs.put("cert1", cert1); + certs.put("cert2", cert2); + createTrustStore(truststoreLocation, "password", certs); + + // and wait to be sure reload has taken place + assertEquals(10, tm.getReloadInterval()); + + // Wait so that the file modification time is different + Thread.sleep((tm.getReloadInterval() + 200)); + + assertEquals(2, tm.getAcceptedIssuers().length); + } finally { + tm.destroy(); + } + } + + @Test + public void testReloadMissingTrustStore() throws Exception { + KeyPair kp = generateKeyPair("RSA"); + cert1 = generateCertificate("CN=Cert1", kp, 30, "SHA1withRSA"); + cert2 = generateCertificate("CN=Cert2", kp, 30, "SHA1withRSA"); + String truststoreLocation = BASEDIR + "/testmissing.jks"; + createTrustStore(truststoreLocation, "password", "cert1", cert1); + + ReloadingX509TrustManager tm = + new ReloadingX509TrustManager("jks", truststoreLocation, "password", 10); + try { + tm.init(); + assertEquals(1, tm.getAcceptedIssuers().length); + X509Certificate cert = tm.getAcceptedIssuers()[0]; + new File(truststoreLocation).delete(); + + // Wait so that the file modification time is different + Thread.sleep((tm.getReloadInterval() + 200)); + + assertEquals(1, tm.getAcceptedIssuers().length); + assertEquals(cert, tm.getAcceptedIssuers()[0]); + } finally { + tm.destroy(); + } + } + + @Test + public void testReloadCorruptTrustStore() throws Exception { + KeyPair kp = generateKeyPair("RSA"); + cert1 = generateCertificate("CN=Cert1", kp, 30, "SHA1withRSA"); + cert2 = generateCertificate("CN=Cert2", kp, 30, "SHA1withRSA"); + String truststoreLocation = BASEDIR + "/testcorrupt.jks"; + createTrustStore(truststoreLocation, "password", "cert1", cert1); + + ReloadingX509TrustManager tm = + new ReloadingX509TrustManager("jks", truststoreLocation, "password", 10); + try { + tm.init(); + assertEquals(1, tm.getAcceptedIssuers().length); + X509Certificate cert = tm.getAcceptedIssuers()[0]; + + OutputStream os = new FileOutputStream(truststoreLocation); + os.write(1); + os.close(); + new File(truststoreLocation).setLastModified(System.currentTimeMillis() - + 1000); + + // Wait so that the file modification time is different + Thread.sleep((tm.getReloadInterval() + 200)); + + assertEquals(1, tm.getAcceptedIssuers().length); + assertEquals(cert, tm.getAcceptedIssuers()[0]); + } finally { + tm.destroy(); + } + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java new file mode 100644 index 00000000000..fa270a1f3a4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security.ssl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.net.URL; +import java.security.GeneralSecurityException; + +public class TestSSLFactory { + + private static final String BASEDIR = + System.getProperty("test.build.dir", "target/test-dir") + "/" + + TestSSLFactory.class.getSimpleName(); + + @BeforeClass + public static void setUp() throws Exception { + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + } + + private Configuration createConfiguration(boolean clientCert) + throws Exception { + Configuration conf = new Configuration(); + String keystoresDir = new File(BASEDIR).getAbsolutePath(); + String sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfsDir, conf, clientCert); + return conf; + } + + @After + @Before + public void cleanUp() throws Exception { + String keystoresDir = new File(BASEDIR).getAbsolutePath(); + String sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class); + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfsDir); + } + + @Test(expected = IllegalStateException.class) + public void clientMode() throws Exception { + Configuration conf = createConfiguration(false); + SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + try { + sslFactory.init(); + Assert.assertNotNull(sslFactory.createSSLSocketFactory()); + Assert.assertNotNull(sslFactory.getHostnameVerifier()); + sslFactory.createSSLServerSocketFactory(); + } finally { + sslFactory.destroy(); + } + } + + private void serverMode(boolean clientCert, boolean socket) throws Exception { + Configuration conf = createConfiguration(clientCert); + SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); + try { + sslFactory.init(); + Assert.assertNotNull(sslFactory.createSSLServerSocketFactory()); + Assert.assertEquals(clientCert, sslFactory.isClientCertRequired()); + if (socket) { + sslFactory.createSSLSocketFactory(); + } else { + sslFactory.getHostnameVerifier(); + } + } finally { + sslFactory.destroy(); + } + } + + + @Test(expected = IllegalStateException.class) + public void serverModeWithoutClientCertsSocket() throws Exception { + serverMode(false, true); + } + + @Test(expected = IllegalStateException.class) + public void serverModeWithClientCertsSocket() throws Exception { + serverMode(true, true); + } + + @Test(expected = IllegalStateException.class) + public void serverModeWithoutClientCertsVerifier() throws Exception { + serverMode(false, false); + } + + @Test(expected = IllegalStateException.class) + public void serverModeWithClientCertsVerifier() throws Exception { + serverMode(true, false); + } + + @Test + public void validHostnameVerifier() throws Exception { + Configuration conf = createConfiguration(false); + conf.unset(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY); + SSLFactory sslFactory = new + SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + Assert.assertEquals("DEFAULT", sslFactory.getHostnameVerifier().toString()); + sslFactory.destroy(); + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL"); + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + Assert.assertEquals("ALLOW_ALL", + sslFactory.getHostnameVerifier().toString()); + sslFactory.destroy(); + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "DEFAULT_AND_LOCALHOST"); + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + Assert.assertEquals("DEFAULT_AND_LOCALHOST", + sslFactory.getHostnameVerifier().toString()); + sslFactory.destroy(); + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "STRICT"); + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + Assert.assertEquals("STRICT", sslFactory.getHostnameVerifier().toString()); + sslFactory.destroy(); + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "STRICT_IE6"); + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + sslFactory.init(); + Assert.assertEquals("STRICT_IE6", + sslFactory.getHostnameVerifier().toString()); + sslFactory.destroy(); + } + + @Test(expected = GeneralSecurityException.class) + public void invalidHostnameVerifier() throws Exception { + Configuration conf = createConfiguration(false); + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "foo"); + SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + try { + sslFactory.init(); + } finally { + sslFactory.destroy(); + } + } + +} diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8bc35a5616d..2cc69e6c78d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -135,6 +135,8 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process command-line. (ahmed via tucu) + MAPREDUCE-4417. add support for encrypted shuffle (tucu) + IMPROVEMENTS MAPREDUCE-4157. ResourceManager should not kill apps that are well behaved diff --git a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml index ff3142cd776..73d54038a5f 100644 --- a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml +++ b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml @@ -473,5 +473,10 @@ - + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index cf5d92b4c7c..5e93fa5a5cf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TypeConverter; @@ -43,6 +44,7 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent; import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent; import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; @@ -108,7 +110,8 @@ public abstract class TaskImpl implements Task, EventHandler { private long scheduledTime; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - + + protected boolean encryptedShuffle; protected Credentials credentials; protected Token jobToken; @@ -274,6 +277,8 @@ public abstract class TaskImpl implements Task, EventHandler { this.jobToken = jobToken; this.metrics = metrics; this.appContext = appContext; + this.encryptedShuffle = conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, + MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT); // See if this is from a previous generation. if (completedTasksFromPreviousRun != null @@ -637,9 +642,10 @@ public abstract class TaskImpl implements Task, EventHandler { TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(-1); - tce.setMapOutputServerAddress("http://" - + attempt.getNodeHttpAddress().split(":")[0] + ":" - + attempt.getShufflePort()); + String scheme = (encryptedShuffle) ? "https://" : "http://"; + tce.setMapOutputServerAddress(scheme + + attempt.getNodeHttpAddress().split(":")[0] + ":" + + attempt.getShufflePort()); tce.setStatus(status); tce.setAttemptId(attempt.getID()); int runTime = 0; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java index 82ee5f00a7f..fb9a1ff6f67 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java @@ -79,4 +79,9 @@ public interface MRConfig { public static final int MAX_BLOCK_LOCATIONS_DEFAULT = 10; public static final String MAX_BLOCK_LOCATIONS_KEY = "mapreduce.job.max.split.locations"; + + public static final String SHUFFLE_SSL_ENABLED_KEY = + "mapreduce.shuffle.ssl.enabled"; + + public static final boolean SHUFFLE_SSL_ENABLED_DEFAULT = false; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java index f3e7fd61c27..64bc43e51b6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java @@ -25,11 +25,13 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.HttpURLConnection; import java.net.URLConnection; +import java.security.GeneralSecurityException; import java.util.HashSet; import java.util.List; import java.util.Set; import javax.crypto.SecretKey; +import javax.net.ssl.HttpsURLConnection; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -42,9 +44,11 @@ import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.IFileInputStream; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.mapreduce.task.reduce.MapOutput.Type; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; @@ -92,6 +96,9 @@ class Fetcher extends Thread { private volatile boolean stopped = false; + private static boolean sslShuffle; + private static SSLFactory sslFactory; + public Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleScheduler scheduler, MergeManager merger, Reporter reporter, ShuffleClientMetrics metrics, @@ -135,6 +142,20 @@ class Fetcher extends Thread { setName("fetcher#" + id); setDaemon(true); + + synchronized (Fetcher.class) { + sslShuffle = job.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, + MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT); + if (sslShuffle && sslFactory == null) { + sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, job); + try { + sslFactory.init(); + } catch (Exception ex) { + sslFactory.destroy(); + throw new RuntimeException(ex); + } + } + } } public void run() { @@ -173,8 +194,25 @@ class Fetcher extends Thread { } catch (InterruptedException ie) { LOG.warn("Got interrupt while joining " + getName(), ie); } + if (sslFactory != null) { + sslFactory.destroy(); + } } + protected HttpURLConnection openConnection(URL url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + if (sslShuffle) { + HttpsURLConnection httpsConn = (HttpsURLConnection) conn; + try { + httpsConn.setSSLSocketFactory(sslFactory.createSSLSocketFactory()); + } catch (GeneralSecurityException ex) { + throw new IOException(ex); + } + httpsConn.setHostnameVerifier(sslFactory.getHostnameVerifier()); + } + return conn; + } + /** * The crux of the matter... * @@ -205,7 +243,7 @@ class Fetcher extends Thread { try { URL url = getMapOutputURL(host, maps); - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + HttpURLConnection connection = openConnection(url); // generate hash of the url String msgToEncode = SecureShuffleUtils.buildMsgFrom(url); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 8591079b3a8..5fee954bbaf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -512,6 +512,21 @@ single shuffle can consume + + mapreduce.shuffle.ssl.enabled + false + + Whether to use SSL for for the Shuffle HTTP endpoints. + + + + + mapreduce.shuffle.ssl.file.buffer.size + 65536 + Buffer size for reading spills from file when using SSL. + + + mapreduce.reduce.markreset.buffer.percent 0.0 diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java new file mode 100644 index 00000000000..d6a17cf97bd --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce.security.ssl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRClientCluster; +import org.apache.hadoop.mapred.MiniMRClientClusterFactory; +import org.apache.hadoop.mapred.RunningJob; + +import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.Assert; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.net.URL; + +public class TestEncryptedShuffle { + + private static final String BASEDIR = + System.getProperty("test.build.dir", "target/test-dir") + "/" + + TestEncryptedShuffle.class.getSimpleName(); + + @BeforeClass + public static void setUp() throws Exception { + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + } + + @Before + public void createCustomYarnClasspath() throws Exception { + String classpathDir = + KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); + + URL url = Thread.currentThread().getContextClassLoader(). + getResource("mrapp-generated-classpath"); + File f = new File(url.getPath()); + BufferedReader reader = new BufferedReader(new FileReader(f)); + String cp = reader.readLine(); + cp = cp + ":" + classpathDir; + f = new File(classpathDir, "mrapp-generated-classpath"); + Writer writer = new FileWriter(f); + writer.write(cp); + writer.close(); + new File(classpathDir, "core-site.xml").delete(); + } + + @After + public void cleanUpMiniClusterSpecialConfig() throws Exception { + String classpathDir = + KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); + new File(classpathDir, "mrapp-generated-classpath").delete(); + new File(classpathDir, "core-site.xml").delete(); + String keystoresDir = new File(BASEDIR).getAbsolutePath(); + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, classpathDir); + } + + private MiniDFSCluster dfsCluster = null; + private MiniMRClientCluster mrCluster = null; + + private void startCluster(Configuration conf) throws Exception { + if (System.getProperty("hadoop.log.dir") == null) { + System.setProperty("hadoop.log.dir", "target/test-dir"); + } + conf.set("dfs.block.access.token.enable", "false"); + conf.set("dfs.permissions", "true"); + conf.set("hadoop.security.authentication", "simple"); + dfsCluster = new MiniDFSCluster(conf, 1, true, null); + FileSystem fileSystem = dfsCluster.getFileSystem(); + fileSystem.mkdirs(new Path("/tmp")); + fileSystem.mkdirs(new Path("/user")); + fileSystem.mkdirs(new Path("/hadoop/mapred/system")); + fileSystem.setPermission( + new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx")); + fileSystem.setPermission( + new Path("/user"), FsPermission.valueOf("-rwxrwxrwx")); + fileSystem.setPermission( + new Path("/hadoop/mapred/system"), FsPermission.valueOf("-rwx------")); + FileSystem.setDefaultUri(conf, fileSystem.getUri()); + mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf); + + // so the minicluster conf is avail to the containers. + String classpathDir = + KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); + Writer writer = new FileWriter(classpathDir + "/core-site.xml"); + mrCluster.getConfig().writeXml(writer); + writer.close(); + } + + private void stopCluster() throws Exception { + if (mrCluster != null) { + mrCluster.stop(); + } + if (dfsCluster != null) { + dfsCluster.shutdown(); + } + } + + protected JobConf getJobConf() throws IOException { + return new JobConf(mrCluster.getConfig()); + } + + private void encryptedShuffleWithCerts(boolean useClientCerts) + throws Exception { + try { + Configuration conf = new Configuration(); + String keystoresDir = new File(BASEDIR).getAbsolutePath(); + String sslConfsDir = + KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfsDir, conf, + useClientCerts); + conf.setBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, true); + startCluster(conf); + FileSystem fs = FileSystem.get(getJobConf()); + Path inputDir = new Path("input"); + fs.mkdirs(inputDir); + Writer writer = + new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt"))); + writer.write("hello"); + writer.close(); + + Path outputDir = new Path("output", "output"); + + JobConf jobConf = new JobConf(getJobConf()); + jobConf.setInt("mapred.map.tasks", 1); + jobConf.setInt("mapred.map.max.attempts", 1); + jobConf.setInt("mapred.reduce.max.attempts", 1); + jobConf.set("mapred.input.dir", inputDir.toString()); + jobConf.set("mapred.output.dir", outputDir.toString()); + JobClient jobClient = new JobClient(jobConf); + RunningJob runJob = jobClient.submitJob(jobConf); + runJob.waitForCompletion(); + Assert.assertTrue(runJob.isComplete()); + Assert.assertTrue(runJob.isSuccessful()); + } finally { + stopCluster(); + } + } + + @Test + public void encryptedShuffleWithClientCerts() throws Exception { + encryptedShuffleWithCerts(true); + } + + @Test + public void encryptedShuffleWithoutClientCerts() throws Exception { + encryptedShuffleWithCerts(false); + } + +} + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index a0fcefe856a..a5717c99772 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -55,7 +55,9 @@ import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; import org.apache.hadoop.mapreduce.task.reduce.ShuffleHeader; @@ -101,6 +103,8 @@ import org.jboss.netty.handler.codec.http.HttpResponse; import org.jboss.netty.handler.codec.http.HttpResponseEncoder; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.codec.http.QueryStringDecoder; +import org.jboss.netty.handler.ssl.SslHandler; +import org.jboss.netty.handler.stream.ChunkedFile; import org.jboss.netty.handler.stream.ChunkedWriteHandler; import org.jboss.netty.util.CharsetUtil; @@ -114,6 +118,8 @@ public class ShuffleHandler extends AbstractService private int port; private ChannelFactory selector; private final ChannelGroup accepted = new DefaultChannelGroup(); + private HttpPipelineFactory pipelineFact; + private int sslFileBufferSize; public static final String MAPREDUCE_SHUFFLE_SERVICEID = "mapreduce.shuffle"; @@ -126,6 +132,11 @@ public class ShuffleHandler extends AbstractService public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port"; public static final int DEFAULT_SHUFFLE_PORT = 8080; + public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY = + "mapreduce.shuffle.ssl.file.buffer.size"; + + public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024; + @Metrics(about="Shuffle output metrics", context="mapred") static class ShuffleMetrics implements ChannelFutureListener { @Metric("Shuffle output in bytes") @@ -249,7 +260,11 @@ public class ShuffleHandler extends AbstractService public synchronized void start() { Configuration conf = getConfig(); ServerBootstrap bootstrap = new ServerBootstrap(selector); - HttpPipelineFactory pipelineFact = new HttpPipelineFactory(conf); + try { + pipelineFact = new HttpPipelineFactory(conf); + } catch (Exception ex) { + throw new RuntimeException(ex); + } bootstrap.setPipelineFactory(pipelineFact); port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT); Channel ch = bootstrap.bind(new InetSocketAddress(port)); @@ -259,6 +274,9 @@ public class ShuffleHandler extends AbstractService pipelineFact.SHUFFLE.setPort(port); LOG.info(getName() + " listening on port " + port); super.start(); + + sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY, + DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE); } @Override @@ -266,6 +284,7 @@ public class ShuffleHandler extends AbstractService accepted.close().awaitUninterruptibly(10, TimeUnit.SECONDS); ServerBootstrap bootstrap = new ServerBootstrap(selector); bootstrap.releaseExternalResources(); + pipelineFact.destroy(); super.stop(); } @@ -283,22 +302,38 @@ public class ShuffleHandler extends AbstractService class HttpPipelineFactory implements ChannelPipelineFactory { final Shuffle SHUFFLE; + private SSLFactory sslFactory; - public HttpPipelineFactory(Configuration conf) { + public HttpPipelineFactory(Configuration conf) throws Exception { SHUFFLE = new Shuffle(conf); + if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, + MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) { + sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); + sslFactory.init(); + } + } + + public void destroy() { + if (sslFactory != null) { + sslFactory.destroy(); + } } @Override public ChannelPipeline getPipeline() throws Exception { - return Channels.pipeline( - new HttpRequestDecoder(), - new HttpChunkAggregator(1 << 16), - new HttpResponseEncoder(), - new ChunkedWriteHandler(), - SHUFFLE); - // TODO factor security manager into pipeline - // TODO factor out encode/decode to permit binary shuffle - // TODO factor out decode of index to permit alt. models + ChannelPipeline pipeline = Channels.pipeline(); + if (sslFactory != null) { + pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine())); + } + pipeline.addLast("decoder", new HttpRequestDecoder()); + pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16)); + pipeline.addLast("encoder", new HttpResponseEncoder()); + pipeline.addLast("chunking", new ChunkedWriteHandler()); + pipeline.addLast("shuffle", SHUFFLE); + return pipeline; + // TODO factor security manager into pipeline + // TODO factor out encode/decode to permit binary shuffle + // TODO factor out decode of index to permit alt. models } } @@ -483,17 +518,25 @@ public class ShuffleHandler extends AbstractService LOG.info(spillfile + " not found"); return null; } - final FileRegion partition = new DefaultFileRegion( - spill.getChannel(), info.startOffset, info.partLength); - ChannelFuture writeFuture = ch.write(partition); - writeFuture.addListener(new ChannelFutureListener() { - // TODO error handling; distinguish IO/connection failures, - // attribute to appropriate spill output - @Override - public void operationComplete(ChannelFuture future) { - partition.releaseExternalResources(); - } - }); + ChannelFuture writeFuture; + if (ch.getPipeline().get(SslHandler.class) == null) { + final FileRegion partition = new DefaultFileRegion( + spill.getChannel(), info.startOffset, info.partLength); + writeFuture = ch.write(partition); + writeFuture.addListener(new ChannelFutureListener() { + // TODO error handling; distinguish IO/connection failures, + // attribute to appropriate spill output + @Override + public void operationComplete(ChannelFuture future) { + partition.releaseExternalResources(); + } + }); + } else { + // HTTPS cannot be done with zero copy. + writeFuture = ch.write(new ChunkedFile(spill, info.startOffset, + info.partLength, + sslFileBufferSize)); + } metrics.shuffleConnections.incr(); metrics.shuffleOutputBytes.incr(info.partLength); // optimistic return writeFuture; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm new file mode 100644 index 00000000000..e05951c1797 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm @@ -0,0 +1,320 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + Hadoop Map Reduce Next Generation-${project.version} - Encrypted Shuffle + --- + --- + ${maven.build.timestamp} + +Hadoop MapReduce Next Generation - Encrypted Shuffle + + \[ {{{./index.html}Go Back}} \] + +* {Introduction} + + The Encrypted Shuffle capability allows encryption of the MapReduce shuffle + using HTTPS and with optional client authentication (also known as + bi-directional HTTPS, or HTTPS with client certificates). It comprises: + + * A Hadoop configuration setting for toggling the shuffle between HTTP and + HTTPS. + + * A Hadoop configuration settings for specifying the keystore and truststore + properties (location, type, passwords) used by the shuffle service and the + reducers tasks fetching shuffle data. + + * A way to re-load truststores across the cluster (when a node is added or + removed). + +* {Configuration} + +** <> Properties + + To enable encrypted shuffle, set the following properties in core-site.xml of + all nodes in the cluster: + +*--------------------------------------+---------------------+-----------------+ +| <> | <> | <> | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Whether client certificates are required | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | The hostname verifier to provide for HttpsURLConnections. Valid values are: <>, <>, <>, <> and <> | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | The KeyStoresFactory implementation to use | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory | +*--------------------------------------+---------------------+-----------------+ + + <> Currently requiring client certificates should be set to false. + Refer the {{{ClientCertificates}Client Certificates}} section for details. + + <> All these properties should be marked as final in the cluster + configuration files. + +*** Example: + +------ + ... + + hadoop.ssl.require.client.cert + false + true + + + + hadoop.ssl.hostname.verifier + DEFAULT + true + + + + hadoop.ssl.keystores.factory.class + org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory + true + + + + hadoop.ssl.server.conf + ssl-server.xml + true + + + + hadoop.ssl.client.conf + ssl-client.xml + true + + ... +------ + +** <<>> Properties + + To enable encrypted shuffle, set the following property in mapred-site.xml + of all nodes in the cluster: + +*--------------------------------------+---------------------+-----------------+ +| <> | <> | <> | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Whether encrypted shuffle is enabled | +*--------------------------------------+---------------------+-----------------+ + + <> This property should be marked as final in the cluster + configuration files. + +*** Example: + +------ + ... + + mapreduce.shuffle.ssl.enabled + true + true + + ... +------ + + The Linux container executor should be set to prevent job tasks from + reading the server keystore information and gaining access to the shuffle + server certificates. + + Refer to Hadoop Kerberos configuration for details on how to do this. + +* {Keystore and Truststore Settings} + + Currently <<>> is the only <<>> + implementation. The <<>> implementation uses the + following properties, in the <> and <> files, + to configure the keystores and truststores. + +** <<>> (Shuffle server) Configuration: + + The mapred user should own the <> file and have exclusive + read access to it. + +*---------------------------------------------+---------------------+-----------------+ +| <> | <> | <> | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Keystore file type | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Keystore file location. The mapred user should own this file and have exclusive read access to it. | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Keystore file password | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Truststore file type | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Truststore file location. The mapred user should own this file and have exclusive read access to it. | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Truststore file password | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | 10000 | Truststore reload interval, in milliseconds | +*--------------------------------------+----------------------------+-----------------+ + +*** Example: + +------ + + + + + ssl.server.keystore.type + jks + + + ssl.server.keystore.location + ${user.home}/keystores/server-keystore.jks + + + ssl.server.keystore.password + serverfoo + + + + + ssl.server.truststore.type + jks + + + ssl.server.truststore.location + ${user.home}/keystores/truststore.jks + + + ssl.server.truststore.password + clientserverbar + + + ssl.server.truststore.reload.interval + 10000 + + +------ + +** <<>> (Reducer/Fetcher) Configuration: + + The mapred user should own the <> file and it should have + default permissions. + +*---------------------------------------------+---------------------+-----------------+ +| <> | <> | <> | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Keystore file type | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Keystore file location. The mapred user should own this file and it should have default permissions. | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Keystore file password | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | Truststore file type | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Truststore file location. The mapred user should own this file and it should have default permissions. | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | NONE | Truststore file password | +*---------------------------------------------+---------------------+-----------------+ +| <<>> | 10000 | Truststore reload interval, in milliseconds | +*--------------------------------------+----------------------------+-----------------+ + +*** Example: + +------ + + + + + ssl.client.keystore.type + jks + + + ssl.client.keystore.location + ${user.home}/keystores/client-keystore.jks + + + ssl.client.keystore.password + clientfoo + + + + + ssl.client.truststore.type + jks + + + ssl.client.truststore.location + ${user.home}/keystores/truststore.jks + + + ssl.client.truststore.password + clientserverbar + + + ssl.client.truststore.reload.interval + 10000 + + +------ + +* Activating Encrypted Shuffle + + When you have made the above configuration changes, activate Encrypted + Shuffle by re-starting all NodeManagers. + + <> Using encrypted shuffle will incur in a significant + performance impact. Users should profile this and potentially reserve + 1 or more cores for encrypted shuffle. + +* {ClientCertificates} Client Certificates + + Using Client Certificates does not fully ensure that the client is a + reducer task for the job. Currently, Client Certificates (their private key) + keystore files must be readable by all users submitting jobs to the cluster. + This means that a rogue job could read such those keystore files and use + the client certificates in them to establish a secure connection with a + Shuffle server. However, unless the rogue job has a proper JobToken, it won't + be able to retrieve shuffle data from the Shuffle server. A job, using its + own JobToken, can only retrieve shuffle data that belongs to itself. + +* Reloading Truststores + + By default the truststores will reload their configuration every 10 seconds. + If a new truststore file is copied over the old one, it will be re-read, + and its certificates will replace the old ones. This mechanism is useful for + adding or removing nodes from the cluster, or for adding or removing trusted + clients. In these cases, the client or NodeManager certificate is added to + (or removed from) all the truststore files in the system, and the new + configuration will be picked up without you having to restart the NodeManager + daemons. + +* Debugging + + <> Enable debugging only for troubleshooting, and then only for jobs + running on small amounts of data. It is very verbose and slows down jobs by + several orders of magnitude. (You might need to increase mapred.task.timeout + to prevent jobs from failing because tasks run so slowly.) + + To enable SSL debugging in the reducers, set <<<-Djavax.net.debug=all>>> in + the <<>> property; for example: + +------ + + mapred.reduce.child.java.opts + -Xmx-200m -Djavax.net.debug=all + +------ + + You can do this on a per-job basis, or by means of a cluster-wide setting in + the <<>> file. + + To set this property in NodeManager, set it in the <<>> file: + +------ + YARN_NODEMANAGER_OPTS="-Djavax.net.debug=all $YARN_NODEMANAGER_OPTS" +------ diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm index dd086474982..badd9155509 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm @@ -51,3 +51,5 @@ MapReduce NextGen aka YARN aka MRv2 * {{{./CLIMiniCluster.html}CLI MiniCluster}} + * {{{./EncryptedShuffle.html}Encrypted Shuffle}} + diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 55333b1f883..f449b5d9b40 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -66,6 +66,7 @@ + From 4d4560189adccb941a3dc5eee7add134adbf6519 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 26 Jul 2012 13:39:05 +0000 Subject: [PATCH 21/39] HDFS-3113. httpfs does not support delegation tokens. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365988 13f79535-47bb-0310-9956-ffa450edef68 --- .../dev-support/findbugsExcludeFile.xml | 5 + .../hadoop-hdfs-httpfs/pom.xml | 58 ++ .../fs/http/client/HttpFSFileSystem.java | 283 +++++----- .../client/HttpFSKerberosAuthenticator.java | 226 ++++++++ ...or.java => HttpFSPseudoAuthenticator.java} | 2 +- .../hadoop/fs/http/client/HttpFSUtils.java | 148 +++++ ...r.java => HttpFSAuthenticationFilter.java} | 9 +- .../HttpFSKerberosAuthenticationHandler.java | 255 +++++++++ .../fs/http/server/HttpFSServerWebApp.java | 2 +- .../hadoop/lib/server/ServerException.java | 6 +- .../service/DelegationTokenIdentifier.java | 57 ++ .../lib/service/DelegationTokenManager.java | 76 +++ .../DelegationTokenManagerException.java | 49 ++ .../DelegationTokenManagerService.java | 231 ++++++++ .../hadoop/lib/servlet/ServerWebApp.java | 65 +++ .../src/main/resources/httpfs-default.xml | 31 +- .../src/main/webapp/WEB-INF/web.xml | 2 +- .../fs/http/client/TestHttpFSFileSystem.java | 18 +- .../client/TestHttpFSFileSystem.java.orig | 513 ++++++++++++++++++ .../fs/http/client/TestWebhdfsFileSystem.java | 16 +- ...berosAuthenticationHandlerForTesting.java} | 30 +- ...stHttpFSKerberosAuthenticationHandler.java | 310 +++++++++++ .../fs/http/server/TestHttpFSServer.java | 129 ++++- .../fs/http/server/TestHttpFSServer.java.orig | 236 ++++++++ .../http/server/TestHttpFSWithKerberos.java | 291 ++++++++++ .../TestDelegationTokenManagerService.java | 83 +++ .../apache/hadoop/test/KerberosTestUtils.java | 138 +++++ .../org/apache/hadoop/test/TestDirHelper.java | 4 +- .../apache/hadoop/test/TestJettyHelper.java | 25 +- .../hadoop/test/TestJettyHelper.java.orig | 118 ++++ .../src/test/resources/krb5.conf | 28 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + 32 files changed, 3252 insertions(+), 194 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java rename hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/{HttpPseudoAuthenticator.java => HttpFSPseudoAuthenticator.java} (95%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java rename hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/{AuthFilter.java => HttpFSAuthenticationFilter.java} (92%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenIdentifier.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManager.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManagerException.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/DelegationTokenManagerService.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig rename hadoop-hdfs-project/hadoop-hdfs-httpfs/src/{main/java/org/apache/hadoop/fs/http/client/HttpKerberosAuthenticator.java => test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java} (55%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestDelegationTokenManagerService.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/KerberosTestUtils.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig create mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/krb5.conf diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml index 94c1d76bf3e..b72f5bc26e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml @@ -25,4 +25,9 @@ + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index b40044aff5c..a3838b8efa6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -43,6 +43,8 @@ ${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/httpfs/tomcat + LOCALHOST + **/TestHttpFSWithKerberos.java @@ -267,6 +269,22 @@ + + + ${basedir}/src/test/resources + false + + krb5.conf + + + + ${basedir}/src/test/resources + true + + krb5.conf + + + @@ -281,6 +299,16 @@ maven-surefire-plugin 1 + 600 + + ${project.build.directory}/test-classes/krb5.conf + ${kerberos.realm} + + + **/${test.exclude}.java + ${test.exclude.pattern} + ${test.exclude.kerberos.test} + @@ -395,6 +423,36 @@ + + testKerberos + + false + + + _ + + + + + org.apache.maven.plugins + maven-surefire-plugin + + once + 600 + + ${project.build.directory}/test-classes/krb5.conf + ${kerberos.realm} + localhost + + + **/TestHttpFSWithKerberos.java + + + + + + + docs diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 9191129e110..a5d4a467ce9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.http.client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.DelegationTokenRenewer; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; @@ -28,16 +29,18 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PositionedReadable; import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -47,30 +50,32 @@ import java.io.FileNotFoundException; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; -import java.lang.reflect.Constructor; import java.net.HttpURLConnection; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.net.URLEncoder; +import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; /** * HttpFSServer implementation of the FileSystemAccess FileSystem. *

* This implementation allows a user to access HDFS over HTTP via a HttpFSServer server. */ -public class HttpFSFileSystem extends FileSystem { +public class HttpFSFileSystem extends FileSystem + implements DelegationTokenRenewer.Renewable { - public static final String SERVICE_NAME = "/webhdfs"; + public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME; - public static final String SERVICE_VERSION = "/v1"; + public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION; - public static final String SERVICE_PREFIX = SERVICE_NAME + SERVICE_VERSION; + public static final String SCHEME = "webhdfs"; public static final String OP_PARAM = "op"; public static final String DO_AS_PARAM = "doas"; @@ -84,7 +89,6 @@ public class HttpFSFileSystem extends FileSystem { public static final String GROUP_PARAM = "group"; public static final String MODIFICATION_TIME_PARAM = "modificationtime"; public static final String ACCESS_TIME_PARAM = "accesstime"; - public static final String RENEWER_PARAM = "renewer"; public static final Short DEFAULT_PERMISSION = 0755; @@ -144,9 +148,6 @@ public class HttpFSFileSystem extends FileSystem { public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed"; public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota"; - public static final String DELEGATION_TOKEN_JSON = "Token"; - public static final String DELEGATION_TOKEN_URL_STRING_JSON = "urlString"; - public static final String ERROR_JSON = "RemoteException"; public static final String ERROR_EXCEPTION_JSON = "exception"; public static final String ERROR_CLASSNAME_JSON = "javaClassName"; @@ -184,8 +185,31 @@ public class HttpFSFileSystem extends FileSystem { private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); private URI uri; + private InetSocketAddress httpFSAddr; private Path workingDir; + private UserGroupInformation realUser; private String doAs; + private Token delegationToken; + + //This method enables handling UGI doAs with SPNEGO, we have to + //fallback to the realuser who logged in with Kerberos credentials + private T doAsRealUserIfNecessary(final Callable callable) + throws IOException { + try { + if (realUser.getShortUserName().equals(doAs)) { + return callable.call(); + } else { + return realUser.doAs(new PrivilegedExceptionAction() { + @Override + public T run() throws Exception { + return callable.call(); + } + }); + } + } catch (Exception ex) { + throw new IOException(ex.toString(), ex); + } + } /** * Convenience method that creates a HttpURLConnection for the @@ -204,25 +228,23 @@ public class HttpFSFileSystem extends FileSystem { * * @throws IOException thrown if an IO error occurrs. */ - private HttpURLConnection getConnection(String method, Map params, - Path path, boolean makeQualified) throws IOException { - params.put(DO_AS_PARAM, doAs); + private HttpURLConnection getConnection(final String method, + Map params, Path path, boolean makeQualified) + throws IOException { + if (!realUser.getShortUserName().equals(doAs)) { + params.put(DO_AS_PARAM, doAs); + } + HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken); if (makeQualified) { path = makeQualified(path); } - URI uri = path.toUri(); - StringBuilder sb = new StringBuilder(); - sb.append(uri.getScheme()).append("://").append(uri.getAuthority()). - append(SERVICE_PREFIX).append(uri.getPath()); - - String separator = "?"; - for (Map.Entry entry : params.entrySet()) { - sb.append(separator).append(entry.getKey()).append("="). - append(URLEncoder.encode(entry.getValue(), "UTF8")); - separator = "&"; - } - URL url = new URL(sb.toString()); - return getConnection(url, method); + final URL url = HttpFSUtils.createHttpURL(path, params); + return doAsRealUserIfNecessary(new Callable() { + @Override + public HttpURLConnection call() throws Exception { + return getConnection(url, method); + } + }); } /** @@ -240,7 +262,8 @@ public class HttpFSFileSystem extends FileSystem { */ private HttpURLConnection getConnection(URL url, String method) throws IOException { Class klass = - getConf().getClass("httpfs.authenticator.class", HttpKerberosAuthenticator.class, Authenticator.class); + getConf().getClass("httpfs.authenticator.class", + HttpFSKerberosAuthenticator.class, Authenticator.class); Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf()); try { HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken); @@ -254,63 +277,6 @@ public class HttpFSFileSystem extends FileSystem { } } - /** - * Convenience method that JSON Parses the InputStream of a HttpURLConnection. - * - * @param conn the HttpURLConnection. - * - * @return the parsed JSON object. - * - * @throws IOException thrown if the InputStream could not be JSON parsed. - */ - private static Object jsonParse(HttpURLConnection conn) throws IOException { - try { - JSONParser parser = new JSONParser(); - return parser.parse(new InputStreamReader(conn.getInputStream())); - } catch (ParseException ex) { - throw new IOException("JSON parser error, " + ex.getMessage(), ex); - } - } - - /** - * Validates the status of an HttpURLConnection against an expected HTTP - * status code. If the current status code is not the expected one it throws an exception - * with a detail message using Server side error messages if available. - * - * @param conn the HttpURLConnection. - * @param expected the expected HTTP status code. - * - * @throws IOException thrown if the current status code does not match the expected one. - */ - private static void validateResponse(HttpURLConnection conn, int expected) throws IOException { - int status = conn.getResponseCode(); - if (status != expected) { - try { - JSONObject json = (JSONObject) jsonParse(conn); - json = (JSONObject) json.get(ERROR_JSON); - String message = (String) json.get(ERROR_MESSAGE_JSON); - String exception = (String) json.get(ERROR_EXCEPTION_JSON); - String className = (String) json.get(ERROR_CLASSNAME_JSON); - - try { - ClassLoader cl = HttpFSFileSystem.class.getClassLoader(); - Class klass = cl.loadClass(className); - Constructor constr = klass.getConstructor(String.class); - throw (IOException) constr.newInstance(message); - } catch (IOException ex) { - throw ex; - } catch (Exception ex) { - throw new IOException(MessageFormat.format("{0} - {1}", exception, message)); - } - } catch (IOException ex) { - if (ex.getCause() instanceof IOException) { - throw (IOException) ex.getCause(); - } - throw new IOException(MessageFormat.format("HTTP status [{0}], {1}", status, conn.getResponseMessage())); - } - } - } - /** * Called after a new FileSystem instance is constructed. * @@ -320,15 +286,28 @@ public class HttpFSFileSystem extends FileSystem { @Override public void initialize(URI name, Configuration conf) throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - doAs = ugi.getUserName(); + + //the real use is the one that has the Kerberos credentials needed for + //SPNEGO to work + realUser = ugi.getRealUser(); + if (realUser == null) { + realUser = UserGroupInformation.getLoginUser(); + } + doAs = ugi.getShortUserName(); super.initialize(name, conf); try { - uri = new URI(name.getScheme() + "://" + name.getHost() + ":" + name.getPort()); + uri = new URI(name.getScheme() + "://" + name.getAuthority()); + httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString()); } catch (URISyntaxException ex) { throw new IOException(ex); } } + @Override + public String getScheme() { + return SCHEME; + } + /** * Returns a URI whose scheme and authority identify this FileSystem. * @@ -339,6 +318,16 @@ public class HttpFSFileSystem extends FileSystem { return uri; } + /** + * Get the default port for this file system. + * @return the default port or 0 if there isn't one + */ + @Override + protected int getDefaultPort() { + return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); + } + /** * HttpFSServer subclass of the FSDataInputStream. *

@@ -397,7 +386,7 @@ public class HttpFSFileSystem extends FileSystem { params.put(OP_PARAM, Operation.OPEN.toString()); HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params, f, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); return new FSDataInputStream( new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); } @@ -424,7 +413,7 @@ public class HttpFSFileSystem extends FileSystem { try { super.close(); } finally { - validateResponse(conn, closeStatus); + HttpFSUtils.validateResponse(conn, closeStatus); } } @@ -460,11 +449,11 @@ public class HttpFSFileSystem extends FileSystem { OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics); } catch (IOException ex) { - validateResponse(conn, expectedStatus); + HttpFSUtils.validateResponse(conn, expectedStatus); throw ex; } } else { - validateResponse(conn, HTTP_TEMPORARY_REDIRECT); + HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]"); } } else { @@ -476,7 +465,7 @@ public class HttpFSFileSystem extends FileSystem { if (exceptionAlreadyHandled) { throw ex; } else { - validateResponse(conn, HTTP_TEMPORARY_REDIRECT); + HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); throw ex; } } @@ -548,8 +537,8 @@ public class HttpFSFileSystem extends FileSystem { params.put(DESTINATION_PARAM, dst.toString()); HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(), params, src, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) jsonParse(conn); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); return (Boolean) json.get(RENAME_JSON); } @@ -584,8 +573,8 @@ public class HttpFSFileSystem extends FileSystem { params.put(RECURSIVE_PARAM, Boolean.toString(recursive)); HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(), params, f, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) jsonParse(conn); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); return (Boolean) json.get(DELETE_JSON); } @@ -605,8 +594,8 @@ public class HttpFSFileSystem extends FileSystem { params.put(OP_PARAM, Operation.LISTSTATUS.toString()); HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(), params, f, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) jsonParse(conn); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); json = (JSONObject) json.get(FILE_STATUSES_JSON); JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON); FileStatus[] array = new FileStatus[jsonArray.size()]; @@ -653,8 +642,8 @@ public class HttpFSFileSystem extends FileSystem { params.put(PERMISSION_PARAM, permissionToString(permission)); HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(), params, f, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) jsonParse(conn); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); return (Boolean) json.get(MKDIRS_JSON); } @@ -674,8 +663,8 @@ public class HttpFSFileSystem extends FileSystem { params.put(OP_PARAM, Operation.GETFILESTATUS.toString()); HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(), params, f, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) jsonParse(conn); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); json = (JSONObject) json.get(FILE_STATUS_JSON); f = makeQualified(f); return createFileStatus(f, json); @@ -693,8 +682,8 @@ public class HttpFSFileSystem extends FileSystem { HttpURLConnection conn = getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params, new Path(getUri().toString(), "/"), false); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) jsonParse(conn); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); return new Path((String) json.get(HOME_DIR_JSON)); } catch (IOException ex) { throw new RuntimeException(ex); @@ -718,7 +707,7 @@ public class HttpFSFileSystem extends FileSystem { params.put(GROUP_PARAM, groupname); HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(), params, p, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); } /** @@ -733,7 +722,7 @@ public class HttpFSFileSystem extends FileSystem { params.put(OP_PARAM, Operation.SETPERMISSION.toString()); params.put(PERMISSION_PARAM, permissionToString(permission)); HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); } /** @@ -755,7 +744,7 @@ public class HttpFSFileSystem extends FileSystem { params.put(ACCESS_TIME_PARAM, Long.toString(atime)); HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(), params, p, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); } /** @@ -777,19 +766,11 @@ public class HttpFSFileSystem extends FileSystem { params.put(REPLICATION_PARAM, Short.toString(replication)); HttpURLConnection conn = getConnection(Operation.SETREPLICATION.getMethod(), params, src, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = (JSONObject) jsonParse(conn); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); return (Boolean) json.get(SET_REPLICATION_JSON); } - /** - * Creates a FileStatus object using a JSON file-status payload - * received from a HttpFSServer server. - * - * @param json a JSON file-status payload received from a HttpFSServer server - * - * @return the corresponding FileStatus - */ private FileStatus createFileStatus(Path parent, JSONObject json) { String pathSuffix = (String) json.get(PATH_SUFFIX_JSON); Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix); @@ -828,9 +809,9 @@ public class HttpFSFileSystem extends FileSystem { params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString()); HttpURLConnection conn = getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - JSONObject json = - (JSONObject) ((JSONObject) jsonParse(conn)).get(CONTENT_SUMMARY_JSON); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) ((JSONObject) + HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON); return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON), (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON), (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON), @@ -846,9 +827,9 @@ public class HttpFSFileSystem extends FileSystem { params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString()); HttpURLConnection conn = getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true); - validateResponse(conn, HttpURLConnection.HTTP_OK); - final JSONObject json = - (JSONObject) ((JSONObject) jsonParse(conn)).get(FILE_CHECKSUM_JSON); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + final JSONObject json = (JSONObject) ((JSONObject) + HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON); return new FileChecksum() { @Override public String getAlgorithmName() { @@ -877,4 +858,56 @@ public class HttpFSFileSystem extends FileSystem { }; } + + @Override + @SuppressWarnings("deprecation") + public Token getDelegationToken(final String renewer) + throws IOException { + return doAsRealUserIfNecessary(new Callable>() { + @Override + public Token call() throws Exception { + return HttpFSKerberosAuthenticator. + getDelegationToken(uri, httpFSAddr, authToken, renewer); + } + }); + } + + + @Override + public List> getDelegationTokens(final String renewer) + throws IOException { + return doAsRealUserIfNecessary(new Callable>>() { + @Override + public List> call() throws Exception { + return HttpFSKerberosAuthenticator. + getDelegationTokens(uri, httpFSAddr, authToken, renewer); + } + }); + } + + public long renewDelegationToken(final Token token) throws IOException { + return doAsRealUserIfNecessary(new Callable() { + @Override + public Long call() throws Exception { + return HttpFSKerberosAuthenticator. + renewDelegationToken(uri, authToken, token); + } + }); + } + + public void cancelDelegationToken(final Token token) throws IOException { + HttpFSKerberosAuthenticator. + cancelDelegationToken(uri, authToken, token); + } + + @Override + public Token getRenewToken() { + return delegationToken; + } + + @Override + public void setDelegationToken(Token token) { + delegationToken = token; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java new file mode 100644 index 00000000000..f73ed7e3b00 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.http.client; + + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A KerberosAuthenticator subclass that fallback to + * {@link HttpFSPseudoAuthenticator}. + */ +public class HttpFSKerberosAuthenticator extends KerberosAuthenticator { + + /** + * Returns the fallback authenticator if the server does not use + * Kerberos SPNEGO HTTP authentication. + * + * @return a {@link HttpFSPseudoAuthenticator} instance. + */ + @Override + protected Authenticator getFallBackAuthenticator() { + return new HttpFSPseudoAuthenticator(); + } + + private static final String HTTP_GET = "GET"; + private static final String HTTP_PUT = "PUT"; + + public static final String DELEGATION_PARAM = "delegation"; + public static final String TOKEN_PARAM = "token"; + public static final String RENEWER_PARAM = "renewer"; + public static final String TOKEN_KIND = "HTTPFS_DELEGATION_TOKEN"; + public static final String DELEGATION_TOKEN_JSON = "Token"; + public static final String DELEGATION_TOKENS_JSON = "Tokens"; + public static final String DELEGATION_TOKEN_URL_STRING_JSON = "urlString"; + public static final String RENEW_DELEGATION_TOKEN_JSON = "long"; + + /** + * DelegationToken operations. + */ + public static enum DelegationTokenOperation { + GETDELEGATIONTOKEN(HTTP_GET, true), + GETDELEGATIONTOKENS(HTTP_GET, true), + RENEWDELEGATIONTOKEN(HTTP_PUT, true), + CANCELDELEGATIONTOKEN(HTTP_PUT, false); + + private String httpMethod; + private boolean requiresKerberosCredentials; + + private DelegationTokenOperation(String httpMethod, + boolean requiresKerberosCredentials) { + this.httpMethod = httpMethod; + this.requiresKerberosCredentials = requiresKerberosCredentials; + } + + public String getHttpMethod() { + return httpMethod; + } + + public boolean requiresKerberosCredentials() { + return requiresKerberosCredentials; + } + + } + + public static void injectDelegationToken(Map params, + Token dtToken) + throws IOException { + if (dtToken != null) { + params.put(DELEGATION_PARAM, dtToken.encodeToUrlString()); + } + } + + private boolean hasDelegationToken(URL url) { + return url.getQuery().contains(DELEGATION_PARAM + "="); + } + + @Override + public void authenticate(URL url, AuthenticatedURL.Token token) + throws IOException, AuthenticationException { + if (!hasDelegationToken(url)) { + super.authenticate(url, token); + } + } + + public static final String OP_PARAM = "op"; + + private static List> getDelegationTokens(URI fsURI, + InetSocketAddress httpFSAddr, DelegationTokenOperation op, + AuthenticatedURL.Token token, String renewer) + throws IOException { + Map params = new HashMap(); + params.put(OP_PARAM, op.toString()); + params.put(RENEWER_PARAM,renewer); + URL url = HttpFSUtils.createHttpURL(new Path(fsURI), params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new HttpFSKerberosAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod(op.getHttpMethod()); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + List list = new ArrayList(); + if (op == DelegationTokenOperation.GETDELEGATIONTOKEN) { + JSONObject json = (JSONObject) ((JSONObject) + HttpFSUtils.jsonParse(conn)).get(DELEGATION_TOKEN_JSON); + String tokenStr = (String) + json.get(DELEGATION_TOKEN_URL_STRING_JSON); + list.add(tokenStr); + } + else if (op == DelegationTokenOperation.GETDELEGATIONTOKENS) { + JSONObject json = (JSONObject) ((JSONObject) + HttpFSUtils.jsonParse(conn)).get(DELEGATION_TOKENS_JSON); + JSONArray array = (JSONArray) json.get(DELEGATION_TOKEN_JSON); + for (Object element : array) { + String tokenStr = (String) + ((Map) element).get(DELEGATION_TOKEN_URL_STRING_JSON); + list.add(tokenStr); + } + + } else { + throw new IllegalArgumentException("Invalid operation: " + + op.toString()); + } + List> dTokens = new ArrayList>(); + for (String tokenStr : list) { + Token dToken = + new Token(); + dToken.decodeFromUrlString(tokenStr); + dTokens.add(dToken); + SecurityUtil.setTokenService(dToken, httpFSAddr); + } + return dTokens; + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static List> getDelegationTokens(URI fsURI, + InetSocketAddress httpFSAddr, AuthenticatedURL.Token token, + String renewer) throws IOException { + return getDelegationTokens(fsURI, httpFSAddr, + DelegationTokenOperation.GETDELEGATIONTOKENS, token, renewer); + } + + public static Token getDelegationToken(URI fsURI, + InetSocketAddress httpFSAddr, AuthenticatedURL.Token token, + String renewer) throws IOException { + return getDelegationTokens(fsURI, httpFSAddr, + DelegationTokenOperation.GETDELEGATIONTOKENS, token, renewer).get(0); + } + + public static long renewDelegationToken(URI fsURI, + AuthenticatedURL.Token token, Token dToken) throws IOException { + Map params = new HashMap(); + params.put(OP_PARAM, + DelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + params.put(TOKEN_PARAM, dToken.encodeToUrlString()); + URL url = HttpFSUtils.createHttpURL(new Path(fsURI), params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new HttpFSKerberosAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod( + DelegationTokenOperation.RENEWDELEGATIONTOKEN.getHttpMethod()); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) ((JSONObject) + HttpFSUtils.jsonParse(conn)).get(DELEGATION_TOKEN_JSON); + return (Long)(json.get(RENEW_DELEGATION_TOKEN_JSON)); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static void cancelDelegationToken(URI fsURI, + AuthenticatedURL.Token token, Token dToken) throws IOException { + Map params = new HashMap(); + params.put(OP_PARAM, + DelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + params.put(TOKEN_PARAM, dToken.encodeToUrlString()); + URL url = HttpFSUtils.createHttpURL(new Path(fsURI), params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new HttpFSKerberosAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod( + DelegationTokenOperation.CANCELDELEGATIONTOKEN.getHttpMethod()); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpPseudoAuthenticator.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSPseudoAuthenticator.java similarity index 95% rename from hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpPseudoAuthenticator.java rename to hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSPseudoAuthenticator.java index 9ac75a0aec9..927b1aa188d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpPseudoAuthenticator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSPseudoAuthenticator.java @@ -27,7 +27,7 @@ import java.io.IOException; * A PseudoAuthenticator subclass that uses FileSystemAccess's * UserGroupInformation to obtain the client user name (the UGI's login user). */ -public class HttpPseudoAuthenticator extends PseudoAuthenticator { +public class HttpFSPseudoAuthenticator extends PseudoAuthenticator { /** * Return the client user name. diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java new file mode 100644 index 00000000000..5a8f8c78fe6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.http.client; + +import org.apache.hadoop.fs.Path; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.util.Map; + +/** + * Utility methods used by HttpFS classes. + */ +public class HttpFSUtils { + + public static final String SERVICE_NAME = "/webhdfs"; + + public static final String SERVICE_VERSION = "/v1"; + + private static final String SERVICE_PATH = SERVICE_NAME + SERVICE_VERSION; + + /** + * Convenience method that creates an HTTP URL for the + * HttpFSServer file system operations. + *

+ * + * @param path the file path. + * @param params the query string parameters. + * + * @return a URL for the HttpFSServer server, + * + * @throws IOException thrown if an IO error occurrs. + */ + static URL createHttpURL(Path path, Map params) + throws IOException { + URI uri = path.toUri(); + String realScheme; + if (uri.getScheme().equalsIgnoreCase(HttpFSFileSystem.SCHEME)) { + realScheme = "http"; + } else { + throw new IllegalArgumentException(MessageFormat.format( + "Invalid scheme [{0}] it should be 'webhdfs'", uri)); + } + StringBuilder sb = new StringBuilder(); + sb.append(realScheme).append("://").append(uri.getAuthority()). + append(SERVICE_PATH).append(uri.getPath()); + + String separator = "?"; + for (Map.Entry entry : params.entrySet()) { + sb.append(separator).append(entry.getKey()).append("="). + append(URLEncoder.encode(entry.getValue(), "UTF8")); + separator = "&"; + } + return new URL(sb.toString()); + } + + /** + * Validates the status of an HttpURLConnection against an + * expected HTTP status code. If the current status code is not the expected + * one it throws an exception with a detail message using Server side error + * messages if available. + * + * @param conn the HttpURLConnection. + * @param expected the expected HTTP status code. + * + * @throws IOException thrown if the current status code does not match the + * expected one. + */ + @SuppressWarnings({"unchecked", "deprecation"}) + static void validateResponse(HttpURLConnection conn, int expected) + throws IOException { + int status = conn.getResponseCode(); + if (status != expected) { + try { + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); + json = (JSONObject) json.get(HttpFSFileSystem.ERROR_JSON); + String message = (String) json.get(HttpFSFileSystem.ERROR_MESSAGE_JSON); + String exception = (String) + json.get(HttpFSFileSystem.ERROR_EXCEPTION_JSON); + String className = (String) + json.get(HttpFSFileSystem.ERROR_CLASSNAME_JSON); + + try { + ClassLoader cl = HttpFSFileSystem.class.getClassLoader(); + Class klass = cl.loadClass(className); + Constructor constr = klass.getConstructor(String.class); + throw (IOException) constr.newInstance(message); + } catch (IOException ex) { + throw ex; + } catch (Exception ex) { + throw new IOException(MessageFormat.format("{0} - {1}", exception, + message)); + } + } catch (IOException ex) { + if (ex.getCause() instanceof IOException) { + throw (IOException) ex.getCause(); + } + throw new IOException( + MessageFormat.format("HTTP status [{0}], {1}", + status, conn.getResponseMessage())); + } + } + } + + /** + * Convenience method that JSON Parses the InputStream of a + * HttpURLConnection. + * + * @param conn the HttpURLConnection. + * + * @return the parsed JSON object. + * + * @throws IOException thrown if the InputStream could not be + * JSON parsed. + */ + static Object jsonParse(HttpURLConnection conn) throws IOException { + try { + JSONParser parser = new JSONParser(); + return parser.parse(new InputStreamReader(conn.getInputStream())); + } catch (ParseException ex) { + throw new IOException("JSON parser error, " + ex.getMessage(), ex); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/AuthFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java similarity index 92% rename from hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/AuthFilter.java rename to hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java index ab778f6c692..4bcfa8447bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/AuthFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs.http.server; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; - import javax.servlet.FilterConfig; import java.io.FileReader; import java.io.IOException; @@ -31,7 +30,7 @@ import java.util.Properties; * Subclass of hadoop-auth AuthenticationFilter that obtains its configuration * from HttpFSServer's server configuration. */ -public class AuthFilter extends AuthenticationFilter { +public class HttpFSAuthenticationFilter extends AuthenticationFilter { private static final String CONF_PREFIX = "httpfs.authentication."; private static final String SIGNATURE_SECRET_FILE = SIGNATURE_SECRET + ".file"; @@ -63,6 +62,11 @@ public class AuthFilter extends AuthenticationFilter { } } + if (props.getProperty(AUTH_TYPE).equals("kerberos")) { + props.setProperty(AUTH_TYPE, + HttpFSKerberosAuthenticationHandler.class.getName()); + } + String signatureSecretFile = props.getProperty(SIGNATURE_SECRET_FILE, null); if (signatureSecretFile == null) { throw new RuntimeException("Undefined property: " + SIGNATURE_SECRET_FILE); @@ -84,5 +88,4 @@ public class AuthFilter extends AuthenticationFilter { return props; } - } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java new file mode 100644 index 00000000000..f8602083e16 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.http.server; + +import org.apache.hadoop.fs.http.client.HttpFSFileSystem; +import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator; +import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator.DelegationTokenOperation; +import org.apache.hadoop.lib.service.DelegationTokenIdentifier; +import org.apache.hadoop.lib.service.DelegationTokenManager; +import org.apache.hadoop.lib.service.DelegationTokenManagerException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.token.Token; +import org.json.simple.JSONObject; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Server side AuthenticationHandler that authenticates requests + * using the incoming delegation token as a 'delegation' query string parameter. + *

+ * If not delegation token is present in the request it delegates to the + * {@link KerberosAuthenticationHandler} + */ +public class HttpFSKerberosAuthenticationHandler + extends KerberosAuthenticationHandler { + + static final Set DELEGATION_TOKEN_OPS = + new HashSet(); + + static { + DELEGATION_TOKEN_OPS.add( + DelegationTokenOperation.GETDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + DelegationTokenOperation.GETDELEGATIONTOKENS.toString()); + DELEGATION_TOKEN_OPS.add( + DelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + DelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + } + + public static final String TYPE = "kerberos-dt"; + + /** + * Returns authentication type of the handler. + * + * @return delegationtoken-kerberos + */ + @Override + public String getType() { + return TYPE; + } + + private static final String ENTER = System.getProperty("line.separator"); + + @Override + @SuppressWarnings("unchecked") + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, HttpServletResponse response) + throws IOException, AuthenticationException { + boolean requestContinues = true; + String op = request.getParameter(HttpFSFileSystem.OP_PARAM); + op = (op != null) ? op.toUpperCase() : null; + if (DELEGATION_TOKEN_OPS.contains(op) && + !request.getMethod().equals("OPTIONS")) { + DelegationTokenOperation dtOp = + DelegationTokenOperation.valueOf(op); + if (dtOp.getHttpMethod().equals(request.getMethod())) { + if (dtOp.requiresKerberosCredentials() && token == null) { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, + MessageFormat.format( + "Operation [{0}] requires SPNEGO authentication established", + dtOp)); + requestContinues = false; + } else { + DelegationTokenManager tokenManager = + HttpFSServerWebApp.get().get(DelegationTokenManager.class); + try { + Map map = null; + switch (dtOp) { + case GETDELEGATIONTOKEN: + case GETDELEGATIONTOKENS: + String renewerParam = + request.getParameter(HttpFSKerberosAuthenticator.RENEWER_PARAM); + if (renewerParam == null) { + renewerParam = token.getUserName(); + } + Token dToken = tokenManager.createToken( + UserGroupInformation.getCurrentUser(), renewerParam); + if (dtOp == DelegationTokenOperation.GETDELEGATIONTOKEN) { + map = delegationTokenToJSON(dToken); + } else { + map = delegationTokensToJSON(Arrays.asList((Token)dToken)); + } + break; + case RENEWDELEGATIONTOKEN: + case CANCELDELEGATIONTOKEN: + String tokenParam = + request.getParameter(HttpFSKerberosAuthenticator.TOKEN_PARAM); + if (tokenParam == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + MessageFormat.format( + "Operation [{0}] requires the parameter [{1}]", + dtOp, HttpFSKerberosAuthenticator.TOKEN_PARAM)); + requestContinues = false; + } else { + if (dtOp == DelegationTokenOperation.CANCELDELEGATIONTOKEN) { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + tokenManager.cancelToken(dt, + UserGroupInformation.getCurrentUser().getUserName()); + } else { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + long expirationTime = + tokenManager.renewToken(dt, token.getUserName()); + map = new HashMap(); + map.put("long", expirationTime); + } + } + break; + } + if (requestContinues) { + response.setStatus(HttpServletResponse.SC_OK); + if (map != null) { + response.setContentType(MediaType.APPLICATION_JSON); + Writer writer = response.getWriter(); + JSONObject.writeJSONString(map, writer); + writer.write(ENTER); + writer.flush(); + + } + requestContinues = false; + } + } catch (DelegationTokenManagerException ex) { + throw new AuthenticationException(ex.toString(), ex); + } + } + } else { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + MessageFormat.format( + "Wrong HTTP method [{0}] for operation [{1}], it should be [{2}]", + request.getMethod(), dtOp, dtOp.getHttpMethod())); + requestContinues = false; + } + } + return requestContinues; + } + + @SuppressWarnings("unchecked") + private static Map delegationTokenToJSON(Token token) throws IOException { + Map json = new LinkedHashMap(); + json.put(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON, + token.encodeToUrlString()); + Map response = new LinkedHashMap(); + response.put(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON, json); + return response; + } + + @SuppressWarnings("unchecked") + private static Map delegationTokensToJSON(List tokens) + throws IOException { + List list = new ArrayList(); + for (Token token : tokens) { + Map map = new HashMap(); + map.put(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON, + token.encodeToUrlString()); + list.add(map); + } + Map map = new HashMap(); + map.put(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON, list); + Map response = new LinkedHashMap(); + response.put(HttpFSKerberosAuthenticator.DELEGATION_TOKENS_JSON, map); + return response; + } + + /** + * Authenticates a request looking for the delegation + * query-string parameter and verifying it is a valid token. If there is not + * delegation query-string parameter, it delegates the + * authentication to the {@link KerberosAuthenticationHandler} unless it is + * disabled. + * + * @param request the HTTP client request. + * @param response the HTTP client response. + * + * @return the authentication token for the authenticated request. + * @throws IOException thrown if an IO error occurred. + * @throws AuthenticationException thrown if the authentication failed. + */ + @Override + public AuthenticationToken authenticate(HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + AuthenticationToken token; + String delegationParam = + request.getParameter(HttpFSKerberosAuthenticator.DELEGATION_PARAM); + if (delegationParam != null) { + try { + Token dt = + new Token(); + dt.decodeFromUrlString(delegationParam); + DelegationTokenManager tokenManager = + HttpFSServerWebApp.get().get(DelegationTokenManager.class); + UserGroupInformation ugi = tokenManager.verifyToken(dt); + final String shortName = ugi.getShortUserName(); + + // creating a ephemeral token + token = new AuthenticationToken(shortName, ugi.getUserName(), + getType()); + token.setExpires(0); + } catch (Throwable ex) { + throw new AuthenticationException("Could not verify DelegationToken, " + + ex.toString(), ex); + } + } else { + token = super.authenticate(request, response); + } + return token; + } + + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java index fec8aa0805b..8a768e89042 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java @@ -70,7 +70,7 @@ public class HttpFSServerWebApp extends ServerWebApp { /** * Constructor used for testing purposes. */ - protected HttpFSServerWebApp(String homeDir, String configDir, String logDir, + public HttpFSServerWebApp(String homeDir, String configDir, String logDir, String tempDir, Configuration config) { super(NAME, homeDir, configDir, logDir, tempDir, config); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java index 2330dcb30cb..1b1476279fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java @@ -39,7 +39,11 @@ public class ServerException extends XException { S08("Could not load service classes, {0}"), S09("Could not set service [{0}] programmatically -server shutting down-, {1}"), S10("Service [{0}] requires service [{1}]"), - S11("Service [{0}] exception during status change to [{1}] -server shutting down-, {2}"); + S11("Service [{0}] exception during status change to [{1}] -server shutting down-, {2}"), + S12("Could not start service [{0}], {1}"), + S13("Missing system property [{0}]"), + S14("Could not initialize server, {0}") + ; private String msg; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenIdentifier.java new file mode 100644 index 00000000000..8d1703fd36a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenIdentifier.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.lib.service; + +import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; + +/** + * HttpFS DelegationTokenIdentifier implementation. + */ +public class DelegationTokenIdentifier + extends AbstractDelegationTokenIdentifier { + + public static final Text KIND_NAME = + new Text(HttpFSKerberosAuthenticator.TOKEN_KIND); + + public DelegationTokenIdentifier() { + } + + /** + * Create a new delegation token identifier + * + * @param owner the effective username of the token owner + * @param renewer the username of the renewer + * @param realUser the real username of the token owner + */ + public DelegationTokenIdentifier(Text owner, Text renewer, Text realUser) { + super(owner, renewer, realUser); + } + + + /** + * Returns the kind, TOKEN_KIND. + * @return returns TOKEN_KIND. + */ + @Override + public Text getKind() { + return KIND_NAME; + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManager.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManager.java new file mode 100644 index 00000000000..4c679576b19 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManager.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.lib.service; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; + +/** + * Service interface to manage HttpFS delegation tokens. + */ +public interface DelegationTokenManager { + + /** + * Creates a delegation token. + * + * @param ugi UGI creating the token. + * @param renewer token renewer. + * @return new delegation token. + * @throws DelegationTokenManagerException thrown if the token could not be + * created. + */ + public Token createToken(UserGroupInformation ugi, + String renewer) + throws DelegationTokenManagerException; + + /** + * Renews a delegation token. + * + * @param token delegation token to renew. + * @param renewer token renewer. + * @return epoc expiration time. + * @throws DelegationTokenManagerException thrown if the token could not be + * renewed. + */ + public long renewToken(Token token, String renewer) + throws DelegationTokenManagerException; + + /** + * Cancels a delegation token. + * + * @param token delegation token to cancel. + * @param canceler token canceler. + * @throws DelegationTokenManagerException thrown if the token could not be + * canceled. + */ + public void cancelToken(Token token, + String canceler) + throws DelegationTokenManagerException; + + /** + * Verifies a delegation token. + * + * @param token delegation token to verify. + * @return the UGI for the token. + * @throws DelegationTokenManagerException thrown if the token could not be + * verified. + */ + public UserGroupInformation verifyToken(Token token) + throws DelegationTokenManagerException; + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManagerException.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManagerException.java new file mode 100644 index 00000000000..0939f217a83 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManagerException.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.lib.service; + +import org.apache.hadoop.lib.lang.XException; + +/** + * Exception thrown by the {@link DelegationTokenManager} service implementation. + */ +public class DelegationTokenManagerException extends XException { + + public enum ERROR implements XException.ERROR { + DT01("Could not verify delegation token, {0}"), + DT02("Could not renew delegation token, {0}"), + DT03("Could not cancel delegation token, {0}"), + DT04("Could not create delegation token, {0}"); + + private String template; + + ERROR(String template) { + this.template = template; + } + + @Override + public String getTemplate() { + return template; + } + } + + public DelegationTokenManagerException(ERROR error, Object... params) { + super(error, params); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/DelegationTokenManagerService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/DelegationTokenManagerService.java new file mode 100644 index 00000000000..3acd1649486 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/DelegationTokenManagerService.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.lib.service.security; + +import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.lib.server.BaseService; +import org.apache.hadoop.lib.server.ServerException; +import org.apache.hadoop.lib.server.ServiceException; +import org.apache.hadoop.lib.service.DelegationTokenIdentifier; +import org.apache.hadoop.lib.service.DelegationTokenManager; +import org.apache.hadoop.lib.service.DelegationTokenManagerException; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; + +/** + * DelegationTokenManager service implementation. + */ +public class DelegationTokenManagerService extends BaseService + implements DelegationTokenManager { + + private static final String PREFIX = "delegation.token.manager"; + + private static final String UPDATE_INTERVAL = "update.interval"; + + private static final String MAX_LIFETIME = "max.lifetime"; + + private static final String RENEW_INTERVAL = "renew.interval"; + + private static final long HOUR = 60 * 60 * 1000; + private static final long DAY = 24 * HOUR; + + DelegationTokenSecretManager secretManager = null; + + public DelegationTokenManagerService() { + super(PREFIX); + } + + /** + * Initializes the service. + * + * @throws ServiceException thrown if the service could not be initialized. + */ + @Override + protected void init() throws ServiceException { + + long updateInterval = getServiceConfig().getLong(UPDATE_INTERVAL, DAY); + long maxLifetime = getServiceConfig().getLong(MAX_LIFETIME, 7 * DAY); + long renewInterval = getServiceConfig().getLong(RENEW_INTERVAL, DAY); + secretManager = new DelegationTokenSecretManager(updateInterval, + maxLifetime, + renewInterval, HOUR); + try { + secretManager.startThreads(); + } catch (IOException ex) { + throw new ServiceException(ServiceException.ERROR.S12, + DelegationTokenManager.class.getSimpleName(), + ex.toString(), ex); + } + } + + /** + * Destroys the service. + */ + @Override + public void destroy() { + secretManager.stopThreads(); + super.destroy(); + } + + /** + * Returns the service interface. + * + * @return the service interface. + */ + @Override + public Class getInterface() { + return DelegationTokenManager.class; + } + + /** + * Creates a delegation token. + * + * @param ugi UGI creating the token. + * @param renewer token renewer. + * @return new delegation token. + * @throws DelegationTokenManagerException thrown if the token could not be + * created. + */ + @Override + public Token createToken(UserGroupInformation ugi, + String renewer) + throws DelegationTokenManagerException { + renewer = (renewer == null) ? ugi.getShortUserName() : renewer; + String user = ugi.getUserName(); + Text owner = new Text(user); + Text realUser = null; + if (ugi.getRealUser() != null) { + realUser = new Text(ugi.getRealUser().getUserName()); + } + DelegationTokenIdentifier tokenIdentifier = + new DelegationTokenIdentifier(owner, new Text(renewer), realUser); + Token token = + new Token(tokenIdentifier, secretManager); + try { + SecurityUtil.setTokenService(token, + HttpFSServerWebApp.get().getAuthority()); + } catch (ServerException ex) { + throw new DelegationTokenManagerException( + DelegationTokenManagerException.ERROR.DT04, ex.toString(), ex); + } + return token; + } + + /** + * Renews a delegation token. + * + * @param token delegation token to renew. + * @param renewer token renewer. + * @return epoc expiration time. + * @throws DelegationTokenManagerException thrown if the token could not be + * renewed. + */ + @Override + public long renewToken(Token token, String renewer) + throws DelegationTokenManagerException { + try { + return secretManager.renewToken(token, renewer); + } catch (IOException ex) { + throw new DelegationTokenManagerException( + DelegationTokenManagerException.ERROR.DT02, ex.toString(), ex); + } + } + + /** + * Cancels a delegation token. + * + * @param token delegation token to cancel. + * @param canceler token canceler. + * @throws DelegationTokenManagerException thrown if the token could not be + * canceled. + */ + @Override + public void cancelToken(Token token, + String canceler) + throws DelegationTokenManagerException { + try { + secretManager.cancelToken(token, canceler); + } catch (IOException ex) { + throw new DelegationTokenManagerException( + DelegationTokenManagerException.ERROR.DT03, ex.toString(), ex); + } + } + + /** + * Verifies a delegation token. + * + * @param token delegation token to verify. + * @return the UGI for the token. + * @throws DelegationTokenManagerException thrown if the token could not be + * verified. + */ + @Override + public UserGroupInformation verifyToken(Token token) + throws DelegationTokenManagerException { + ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); + DataInputStream dis = new DataInputStream(buf); + DelegationTokenIdentifier id = new DelegationTokenIdentifier(); + try { + id.readFields(dis); + dis.close(); + secretManager.verifyToken(id, token.getPassword()); + } catch (Exception ex) { + throw new DelegationTokenManagerException( + DelegationTokenManagerException.ERROR.DT01, ex.toString(), ex); + } + return id.getUser(); + } + + private static class DelegationTokenSecretManager + extends AbstractDelegationTokenSecretManager { + + /** + * Create a secret manager + * + * @param delegationKeyUpdateInterval the number of seconds for rolling new + * secret keys. + * @param delegationTokenMaxLifetime the maximum lifetime of the delegation + * tokens + * @param delegationTokenRenewInterval how often the tokens must be renewed + * @param delegationTokenRemoverScanInterval how often the tokens are + * scanned + * for expired tokens + */ + public DelegationTokenSecretManager(long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, + long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + } + + @Override + public DelegationTokenIdentifier createIdentifier() { + return new DelegationTokenIdentifier(); + } + + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java index 77e31456f7b..277b17fed22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java @@ -18,12 +18,16 @@ package org.apache.hadoop.lib.servlet; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.lib.server.Server; import org.apache.hadoop.lib.server.ServerException; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.text.MessageFormat; /** @@ -36,9 +40,13 @@ public abstract class ServerWebApp extends Server implements ServletContextListe private static final String CONFIG_DIR = ".config.dir"; private static final String LOG_DIR = ".log.dir"; private static final String TEMP_DIR = ".temp.dir"; + private static final String HTTP_HOSTNAME = ".http.hostname"; + private static final String HTTP_PORT = ".http.port"; private static ThreadLocal HOME_DIR_TL = new ThreadLocal(); + private InetSocketAddress authority; + /** * Method for testing purposes. */ @@ -146,6 +154,38 @@ public abstract class ServerWebApp extends Server implements ServletContextListe } } + /** + * Resolves the host & port InetSocketAddress the web server is listening to. + *

+ * This implementation looks for the following 2 properties: + *

    + *
  • #SERVER_NAME#.http.hostname
    • + *
  • #SERVER_NAME#.http.port
    • + *
+ * + * @return the host & port InetSocketAddress the web server is listening to. + * @throws ServerException thrown if any of the above 2 properties is not defined. + */ + protected InetSocketAddress resolveAuthority() throws ServerException { + String hostnameKey = getName() + HTTP_HOSTNAME; + String portKey = getName() + HTTP_PORT; + String host = System.getProperty(hostnameKey); + String port = System.getProperty(portKey); + if (host == null) { + throw new ServerException(ServerException.ERROR.S13, hostnameKey); + } + if (port == null) { + throw new ServerException(ServerException.ERROR.S13, portKey); + } + try { + InetAddress add = InetAddress.getByName(hostnameKey); + int portNum = Integer.parseInt(port); + return new InetSocketAddress(add, portNum); + } catch (UnknownHostException ex) { + throw new ServerException(ServerException.ERROR.S14, ex.toString(), ex); + } + } + /** * Destroys the ServletContextListener which destroys * the Server. @@ -156,4 +196,29 @@ public abstract class ServerWebApp extends Server implements ServletContextListe destroy(); } + /** + * Returns the hostname:port InetSocketAddress the webserver is listening to. + * + * @return the hostname:port InetSocketAddress the webserver is listening to. + */ + public InetSocketAddress getAuthority() throws ServerException { + synchronized (this) { + if (authority == null) { + authority = resolveAuthority(); + } + } + return authority; + } + + /** + * Sets an alternate hostname:port InetSocketAddress to use. + *

+ * For testing purposes. + * + * @param authority alterante authority. + */ + @VisibleForTesting + public void setAuthority(InetSocketAddress authority) { + this.authority = authority; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml index a51f8722708..fc4faf5563e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml @@ -35,6 +35,7 @@ org.apache.hadoop.lib.service.scheduler.SchedulerService, org.apache.hadoop.lib.service.security.GroupsService, org.apache.hadoop.lib.service.security.ProxyUserService, + org.apache.hadoop.lib.service.security.DelegationTokenManagerService, org.apache.hadoop.lib.service.hadoop.FileSystemAccessService @@ -88,12 +89,12 @@ Defines the authentication mechanism used by httpfs for its HTTP clients. - Valid values are 'simple' and 'kerberos'. + Valid values are 'simple' or 'kerberos'. If using 'simple' HTTP clients must specify the username with the 'user.name' query string parameter. - If using 'kerberos' HTTP clients must use HTTP SPNEGO. + If using 'kerberos' HTTP clients must use HTTP SPNEGO or delegation tokens. @@ -153,6 +154,32 @@ + + + + httpfs.delegation.token.manager.update.interval + 86400 + + HttpFS delegation token update interval, default 1 day, in seconds. + + + + + httpfs.delegation.token.manager.max.lifetime + 604800 + + HttpFS delegation token maximum lifetime, default 7 days, in seconds + + + + + httpfs.delegation.token.manager.renewal.interval + 86400 + + HttpFS delegation token update interval, default 1 day, in seconds. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml index 4d5e976fc5a..4c0b3aedb0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml @@ -47,7 +47,7 @@ authFilter - org.apache.hadoop.fs.http.server.AuthFilter + org.apache.hadoop.fs.http.server.HttpFSAuthenticationFilter diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java index ebebd25003f..aaa7356aff5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Writer; +import java.net.URI; import java.net.URL; import java.security.PrivilegedExceptionAction; import java.util.Arrays; @@ -100,16 +101,24 @@ public class TestHttpFSFileSystem extends HFSTestCase { server.start(); } + protected Class getFileSystemClass() { + return HttpFSFileSystem.class; + } + protected FileSystem getHttpFileSystem() throws Exception { Configuration conf = new Configuration(); - conf.set("fs.http.impl", HttpFSFileSystem.class.getName()); - return FileSystem.get(TestJettyHelper.getJettyURL().toURI(), conf); + conf.set("fs.webhdfs.impl", getFileSystemClass().getName()); + URI uri = new URI("webhdfs://" + + TestJettyHelper.getJettyURL().toURI().getAuthority()); + return FileSystem.get(uri, conf); } protected void testGet() throws Exception { FileSystem fs = getHttpFileSystem(); Assert.assertNotNull(fs); - Assert.assertEquals(fs.getUri(), TestJettyHelper.getJettyURL().toURI()); + URI uri = new URI("webhdfs://" + + TestJettyHelper.getJettyURL().toURI().getAuthority()); + Assert.assertEquals(fs.getUri(), uri); fs.close(); } @@ -474,8 +483,9 @@ public class TestHttpFSFileSystem extends HFSTestCase { for (int i = 0; i < Operation.values().length; i++) { ops[i] = new Object[]{Operation.values()[i]}; } + //To test one or a subset of operations do: + //return Arrays.asList(new Object[][]{ new Object[]{Operation.OPEN}}); return Arrays.asList(ops); -// return Arrays.asList(new Object[][]{ new Object[]{Operation.CREATE}}); } private Operation operation; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig new file mode 100644 index 00000000000..ebebd25003f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig @@ -0,0 +1,513 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.http.client; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Writer; +import java.net.URL; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.HFSTestCase; +import org.apache.hadoop.test.HadoopUsersConfTestHelper; +import org.apache.hadoop.test.TestDir; +import org.apache.hadoop.test.TestDirHelper; +import org.apache.hadoop.test.TestHdfs; +import org.apache.hadoop.test.TestHdfsHelper; +import org.apache.hadoop.test.TestJetty; +import org.apache.hadoop.test.TestJettyHelper; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.webapp.WebAppContext; + +@RunWith(value = Parameterized.class) +public class TestHttpFSFileSystem extends HFSTestCase { + + private void createHttpFSServer() throws Exception { + File homeDir = TestDirHelper.getTestDir(); + Assert.assertTrue(new File(homeDir, "conf").mkdir()); + Assert.assertTrue(new File(homeDir, "log").mkdir()); + Assert.assertTrue(new File(homeDir, "temp").mkdir()); + HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath()); + + File secretFile = new File(new File(homeDir, "conf"), "secret"); + Writer w = new FileWriter(secretFile); + w.write("secret"); + w.close(); + + //HDFS configuration + String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); + Configuration conf = new Configuration(false); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName); + File hdfsSite = new File(new File(homeDir, "conf"), "hdfs-site.xml"); + OutputStream os = new FileOutputStream(hdfsSite); + conf.writeXml(os); + os.close(); + + //HTTPFS configuration + conf = new Configuration(false); + conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups", + HadoopUsersConfTestHelper.getHadoopProxyUserGroups()); + conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts", + HadoopUsersConfTestHelper.getHadoopProxyUserHosts()); + conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath()); + File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml"); + os = new FileOutputStream(httpfsSite); + conf.writeXml(os); + os.close(); + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource("webapp"); + WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs"); + Server server = TestJettyHelper.getJettyServer(); + server.addHandler(context); + server.start(); + } + + protected FileSystem getHttpFileSystem() throws Exception { + Configuration conf = new Configuration(); + conf.set("fs.http.impl", HttpFSFileSystem.class.getName()); + return FileSystem.get(TestJettyHelper.getJettyURL().toURI(), conf); + } + + protected void testGet() throws Exception { + FileSystem fs = getHttpFileSystem(); + Assert.assertNotNull(fs); + Assert.assertEquals(fs.getUri(), TestJettyHelper.getJettyURL().toURI()); + fs.close(); + } + + private void testOpen() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + fs.close(); + fs = getHttpFileSystem(); + InputStream is = fs.open(new Path(path.toUri().getPath())); + Assert.assertEquals(is.read(), 1); + is.close(); + fs.close(); + } + + private void testCreate(Path path, boolean override) throws Exception { + FileSystem fs = getHttpFileSystem(); + FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE); + OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024, + (short) 2, 100 * 1024 * 1024, null); + os.write(1); + os.close(); + fs.close(); + + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + FileStatus status = fs.getFileStatus(path); + Assert.assertEquals(status.getReplication(), 2); + Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024); + Assert.assertEquals(status.getPermission(), permission); + InputStream is = fs.open(path); + Assert.assertEquals(is.read(), 1); + is.close(); + fs.close(); + } + + private void testCreate() throws Exception { + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + testCreate(path, false); + testCreate(path, true); + try { + testCreate(path, false); + Assert.fail(); + } catch (IOException ex) { + + } catch (Exception ex) { + Assert.fail(); + } + } + + private void testAppend() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + fs.close(); + fs = getHttpFileSystem(); + os = fs.append(new Path(path.toUri().getPath())); + os.write(2); + os.close(); + fs.close(); + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + InputStream is = fs.open(path); + Assert.assertEquals(is.read(), 1); + Assert.assertEquals(is.read(), 2); + Assert.assertEquals(is.read(), -1); + is.close(); + fs.close(); + } + + private void testRename() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo"); + fs.mkdirs(path); + fs.close(); + fs = getHttpFileSystem(); + Path oldPath = new Path(path.toUri().getPath()); + Path newPath = new Path(path.getParent(), "bar"); + fs.rename(oldPath, newPath); + fs.close(); + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Assert.assertFalse(fs.exists(oldPath)); + Assert.assertTrue(fs.exists(newPath)); + fs.close(); + } + + private void testDelete() throws Exception { + Path foo = new Path(TestHdfsHelper.getHdfsTestDir(), "foo"); + Path bar = new Path(TestHdfsHelper.getHdfsTestDir(), "bar"); + Path foe = new Path(TestHdfsHelper.getHdfsTestDir(), "foe"); + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + fs.mkdirs(foo); + fs.mkdirs(new Path(bar, "a")); + fs.mkdirs(foe); + + FileSystem hoopFs = getHttpFileSystem(); + Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false)); + Assert.assertFalse(fs.exists(foo)); + try { + hoopFs.delete(new Path(bar.toUri().getPath()), false); + Assert.fail(); + } catch (IOException ex) { + } catch (Exception ex) { + Assert.fail(); + } + Assert.assertTrue(fs.exists(bar)); + Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true)); + Assert.assertFalse(fs.exists(bar)); + + Assert.assertTrue(fs.exists(foe)); + Assert.assertTrue(hoopFs.delete(foe, true)); + Assert.assertFalse(fs.exists(foe)); + + hoopFs.close(); + fs.close(); + } + + private void testListStatus() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + FileStatus status1 = fs.getFileStatus(path); + fs.close(); + + fs = getHttpFileSystem(); + FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath())); + fs.close(); + + Assert.assertEquals(status2.getPermission(), status1.getPermission()); + Assert.assertEquals(status2.getPath().toUri().getPath(), status1.getPath().toUri().getPath()); + Assert.assertEquals(status2.getReplication(), status1.getReplication()); + Assert.assertEquals(status2.getBlockSize(), status1.getBlockSize()); + Assert.assertEquals(status2.getAccessTime(), status1.getAccessTime()); + Assert.assertEquals(status2.getModificationTime(), status1.getModificationTime()); + Assert.assertEquals(status2.getOwner(), status1.getOwner()); + Assert.assertEquals(status2.getGroup(), status1.getGroup()); + Assert.assertEquals(status2.getLen(), status1.getLen()); + + FileStatus[] stati = fs.listStatus(path.getParent()); + Assert.assertEquals(stati.length, 1); + Assert.assertEquals(stati[0].getPath().getName(), path.getName()); + } + + private void testWorkingdirectory() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path workingDir = fs.getWorkingDirectory(); + fs.close(); + + fs = getHttpFileSystem(); + Path hoopWorkingDir = fs.getWorkingDirectory(); + fs.close(); + Assert.assertEquals(hoopWorkingDir.toUri().getPath(), workingDir.toUri().getPath()); + + fs = getHttpFileSystem(); + fs.setWorkingDirectory(new Path("/tmp")); + workingDir = fs.getWorkingDirectory(); + fs.close(); + Assert.assertEquals(workingDir.toUri().getPath(), new Path("/tmp").toUri().getPath()); + } + + private void testMkdirs() throws Exception { + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo"); + FileSystem fs = getHttpFileSystem(); + fs.mkdirs(path); + fs.close(); + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Assert.assertTrue(fs.exists(path)); + fs.close(); + } + + private void testSetTimes() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + FileStatus status1 = fs.getFileStatus(path); + fs.close(); + long at = status1.getAccessTime(); + long mt = status1.getModificationTime(); + + fs = getHttpFileSystem(); + fs.setTimes(path, mt + 10, at + 20); + fs.close(); + + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + status1 = fs.getFileStatus(path); + fs.close(); + long atNew = status1.getAccessTime(); + long mtNew = status1.getModificationTime(); + Assert.assertEquals(mtNew, mt + 10); + Assert.assertEquals(atNew, at + 20); + } + + private void testSetPermission() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foodir"); + fs.mkdirs(path); + + fs = getHttpFileSystem(); + FsPermission permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE); + fs.setPermission(path, permission1); + fs.close(); + + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + FileStatus status1 = fs.getFileStatus(path); + fs.close(); + FsPermission permission2 = status1.getPermission(); + Assert.assertEquals(permission2, permission1); + + //sticky bit + fs = getHttpFileSystem(); + permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE, true); + fs.setPermission(path, permission1); + fs.close(); + + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + status1 = fs.getFileStatus(path); + fs.close(); + permission2 = status1.getPermission(); + Assert.assertTrue(permission2.getStickyBit()); + Assert.assertEquals(permission2, permission1); + } + + private void testSetOwner() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + fs.close(); + + fs = getHttpFileSystem(); + String user = HadoopUsersConfTestHelper.getHadoopUsers()[1]; + String group = HadoopUsersConfTestHelper.getHadoopUserGroups(user)[0]; + fs.setOwner(path, user, group); + fs.close(); + + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + FileStatus status1 = fs.getFileStatus(path); + fs.close(); + Assert.assertEquals(status1.getOwner(), user); + Assert.assertEquals(status1.getGroup(), group); + } + + private void testSetReplication() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + fs.close(); + fs.setReplication(path, (short) 2); + + fs = getHttpFileSystem(); + fs.setReplication(path, (short) 1); + fs.close(); + + fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + FileStatus status1 = fs.getFileStatus(path); + fs.close(); + Assert.assertEquals(status1.getReplication(), (short) 1); + } + + private void testChecksum() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + FileChecksum hdfsChecksum = fs.getFileChecksum(path); + fs.close(); + fs = getHttpFileSystem(); + FileChecksum httpChecksum = fs.getFileChecksum(path); + fs.close(); + Assert.assertEquals(httpChecksum.getAlgorithmName(), hdfsChecksum.getAlgorithmName()); + Assert.assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength()); + Assert.assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes()); + } + + private void testContentSummary() throws Exception { + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); + OutputStream os = fs.create(path); + os.write(1); + os.close(); + ContentSummary hdfsContentSummary = fs.getContentSummary(path); + fs.close(); + fs = getHttpFileSystem(); + ContentSummary httpContentSummary = fs.getContentSummary(path); + fs.close(); + Assert.assertEquals(httpContentSummary.getDirectoryCount(), hdfsContentSummary.getDirectoryCount()); + Assert.assertEquals(httpContentSummary.getFileCount(), hdfsContentSummary.getFileCount()); + Assert.assertEquals(httpContentSummary.getLength(), hdfsContentSummary.getLength()); + Assert.assertEquals(httpContentSummary.getQuota(), hdfsContentSummary.getQuota()); + Assert.assertEquals(httpContentSummary.getSpaceConsumed(), hdfsContentSummary.getSpaceConsumed()); + Assert.assertEquals(httpContentSummary.getSpaceQuota(), hdfsContentSummary.getSpaceQuota()); + } + + protected enum Operation { + GET, OPEN, CREATE, APPEND, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, + SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY + } + + private void operation(Operation op) throws Exception { + switch (op) { + case GET: + testGet(); + break; + case OPEN: + testOpen(); + break; + case CREATE: + testCreate(); + break; + case APPEND: + testAppend(); + break; + case RENAME: + testRename(); + break; + case DELETE: + testDelete(); + break; + case LIST_STATUS: + testListStatus(); + break; + case WORKING_DIRECTORY: + testWorkingdirectory(); + break; + case MKDIRS: + testMkdirs(); + break; + case SET_TIMES: + testSetTimes(); + break; + case SET_PERMISSION: + testSetPermission(); + break; + case SET_OWNER: + testSetOwner(); + break; + case SET_REPLICATION: + testSetReplication(); + break; + case CHECKSUM: + testChecksum(); + break; + case CONTENT_SUMMARY: + testContentSummary(); + break; + } + } + + @Parameterized.Parameters + public static Collection operations() { + Object[][] ops = new Object[Operation.values().length][]; + for (int i = 0; i < Operation.values().length; i++) { + ops[i] = new Object[]{Operation.values()[i]}; + } + return Arrays.asList(ops); +// return Arrays.asList(new Object[][]{ new Object[]{Operation.CREATE}}); + } + + private Operation operation; + + public TestHttpFSFileSystem(Operation operation) { + this.operation = operation; + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testOperation() throws Exception { + createHttpFSServer(); + operation(operation); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testOperationDoAs() throws Exception { + createHttpFSServer(); + UserGroupInformation ugi = UserGroupInformation.createProxyUser(HadoopUsersConfTestHelper.getHadoopUsers()[0], + UserGroupInformation.getCurrentUser()); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + operation(operation); + return null; + } + }); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java index 95c005976c1..49aa81854a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java @@ -36,20 +36,8 @@ public class TestWebhdfsFileSystem extends TestHttpFSFileSystem { } @Override - protected FileSystem getHttpFileSystem() throws Exception { - Configuration conf = new Configuration(); - conf.set("fs.webhdfs.impl", WebHdfsFileSystem.class.getName()); - URI uri = new URI("webhdfs://" + TestJettyHelper.getJettyURL().toURI().getAuthority()); - return FileSystem.get(uri, conf); - } - - @Override - protected void testGet() throws Exception { - FileSystem fs = getHttpFileSystem(); - Assert.assertNotNull(fs); - URI uri = new URI("webhdfs://" + TestJettyHelper.getJettyURL().toURI().getAuthority()); - Assert.assertEquals(fs.getUri(), uri); - fs.close(); + protected Class getFileSystemClass() { + return WebHdfsFileSystem.class; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpKerberosAuthenticator.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java similarity index 55% rename from hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpKerberosAuthenticator.java rename to hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java index 8f781bbb767..760cfd548a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpKerberosAuthenticator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java @@ -15,27 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.hadoop.fs.http.server; -package org.apache.hadoop.fs.http.client; +import javax.servlet.ServletException; +import java.util.Properties; +public class HttpFSKerberosAuthenticationHandlerForTesting + extends HttpFSKerberosAuthenticationHandler { -import org.apache.hadoop.security.authentication.client.Authenticator; -import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; - -/** - * A KerberosAuthenticator subclass that fallback to - * {@link HttpPseudoAuthenticator}. - */ -public class HttpKerberosAuthenticator extends KerberosAuthenticator { - - /** - * Returns the fallback authenticator if the server does not use - * Kerberos SPNEGO HTTP authentication. - * - * @return a {@link HttpPseudoAuthenticator} instance. - */ @Override - protected Authenticator getFallBackAuthenticator() { - return new HttpPseudoAuthenticator(); + public void init(Properties config) throws ServletException { + //NOP overwrite to avoid Kerberos initialization + } + + @Override + public void destroy() { + //NOP overwrite to avoid Kerberos initialization } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java new file mode 100644 index 00000000000..588488e830d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java @@ -0,0 +1,310 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.http.server; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.http.client.HttpFSFileSystem; +import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator; +import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator.DelegationTokenOperation; +import org.apache.hadoop.lib.service.DelegationTokenIdentifier; +import org.apache.hadoop.lib.service.DelegationTokenManager; +import org.apache.hadoop.lib.service.DelegationTokenManagerException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationHandler; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.HFSTestCase; +import org.apache.hadoop.test.TestDir; +import org.apache.hadoop.test.TestDirHelper; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; + +public class TestHttpFSKerberosAuthenticationHandler extends HFSTestCase { + + @Test + @TestDir + public void testManagementOperations() throws Exception { + String dir = TestDirHelper.getTestDir().getAbsolutePath(); + + Configuration httpfsConf = new Configuration(false); + HttpFSServerWebApp server = + new HttpFSServerWebApp(dir, dir, dir, dir, httpfsConf); + server.setAuthority(new InetSocketAddress(InetAddress.getLocalHost(), + 14000)); + AuthenticationHandler handler = + new HttpFSKerberosAuthenticationHandlerForTesting(); + try { + server.init(); + handler.init(null); + + testNonManagementOperation(handler); + testManagementOperationErrors(handler); + testGetToken(handler, false, null); + testGetToken(handler, true, null); + testGetToken(handler, false, "foo"); + testGetToken(handler, true, "foo"); + testCancelToken(handler); + testRenewToken(handler); + + } finally { + if (handler != null) { + handler.destroy(); + } + server.destroy(); + } + } + + private void testNonManagementOperation(AuthenticationHandler handler) + throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)). + thenReturn(null); + Assert.assertTrue(handler.managementOperation(null, request, null)); + Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)). + thenReturn(HttpFSFileSystem.Operation.CREATE.toString()); + Assert.assertTrue(handler.managementOperation(null, request, null)); + } + + private void testManagementOperationErrors(AuthenticationHandler handler) + throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)). + thenReturn(DelegationTokenOperation.GETDELEGATIONTOKEN.toString()); + Mockito.when(request.getMethod()).thenReturn("FOO"); + Assert.assertFalse(handler.managementOperation(null, request, response)); + Mockito.verify(response).sendError( + Mockito.eq(HttpServletResponse.SC_BAD_REQUEST), + Mockito.startsWith("Wrong HTTP method")); + + Mockito.reset(response); + Mockito.when(request.getMethod()). + thenReturn(DelegationTokenOperation.GETDELEGATIONTOKEN.getHttpMethod()); + Assert.assertFalse(handler.managementOperation(null, request, response)); + Mockito.verify(response).sendError( + Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), + Mockito.contains("requires SPNEGO")); + } + + private void testGetToken(AuthenticationHandler handler, boolean tokens, + String renewer) + throws Exception { + DelegationTokenOperation op = + (tokens) ? DelegationTokenOperation.GETDELEGATIONTOKENS + : DelegationTokenOperation.GETDELEGATIONTOKEN; + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)). + thenReturn(op.toString()); + Mockito.when(request.getMethod()). + thenReturn(op.getHttpMethod()); + + AuthenticationToken token = Mockito.mock(AuthenticationToken.class); + Mockito.when(token.getUserName()).thenReturn("user"); + Assert.assertFalse(handler.managementOperation(null, request, response)); + Mockito.when(request.getParameter(HttpFSKerberosAuthenticator.RENEWER_PARAM)). + thenReturn(renewer); + + Mockito.reset(response); + StringWriter writer = new StringWriter(); + PrintWriter pwriter = new PrintWriter(writer); + Mockito.when(response.getWriter()).thenReturn(pwriter); + Assert.assertFalse(handler.managementOperation(token, request, response)); + if (renewer == null) { + Mockito.verify(token).getUserName(); + } else { + Mockito.verify(token, Mockito.never()).getUserName(); + } + Mockito.verify(response).setStatus(HttpServletResponse.SC_OK); + Mockito.verify(response).setContentType(MediaType.APPLICATION_JSON); + pwriter.close(); + String responseOutput = writer.toString(); + String tokenLabel = (tokens) + ? HttpFSKerberosAuthenticator.DELEGATION_TOKENS_JSON + : HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON; + if (tokens) { + Assert.assertTrue(responseOutput.contains(tokenLabel)); + } else { + Assert.assertTrue(responseOutput.contains(tokenLabel)); + } + Assert.assertTrue(responseOutput.contains( + HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON)); + JSONObject json = (JSONObject) new JSONParser().parse(responseOutput); + json = (JSONObject) json.get(tokenLabel); + String tokenStr; + if (tokens) { + json = (JSONObject) ((JSONArray) + json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON)).get(0); + } + tokenStr = (String) + json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); + Token dt = new Token(); + dt.decodeFromUrlString(tokenStr); + HttpFSServerWebApp.get().get(DelegationTokenManager.class).verifyToken(dt); + } + + private void testCancelToken(AuthenticationHandler handler) + throws Exception { + DelegationTokenOperation op = + DelegationTokenOperation.CANCELDELEGATIONTOKEN; + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)). + thenReturn(op.toString()); + Mockito.when(request.getMethod()). + thenReturn(op.getHttpMethod()); + + Assert.assertFalse(handler.managementOperation(null, request, response)); + Mockito.verify(response).sendError( + Mockito.eq(HttpServletResponse.SC_BAD_REQUEST), + Mockito.contains("requires the parameter [token]")); + + Mockito.reset(response); + Token token = + HttpFSServerWebApp.get().get(DelegationTokenManager.class).createToken( + UserGroupInformation.getCurrentUser(), "foo"); + Mockito.when(request.getParameter(HttpFSKerberosAuthenticator.TOKEN_PARAM)). + thenReturn(token.encodeToUrlString()); + Assert.assertFalse(handler.managementOperation(null, request, response)); + Mockito.verify(response).setStatus(HttpServletResponse.SC_OK); + try { + HttpFSServerWebApp.get().get(DelegationTokenManager.class).verifyToken(token); + Assert.fail(); + } + catch (DelegationTokenManagerException ex) { + Assert.assertTrue(ex.toString().contains("DT01")); + } + } + + private void testRenewToken(AuthenticationHandler handler) + throws Exception { + DelegationTokenOperation op = + DelegationTokenOperation.RENEWDELEGATIONTOKEN; + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)). + thenReturn(op.toString()); + Mockito.when(request.getMethod()). + thenReturn(op.getHttpMethod()); + + Assert.assertFalse(handler.managementOperation(null, request, response)); + Mockito.verify(response).sendError( + Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), + Mockito.contains("equires SPNEGO authentication established")); + + Mockito.reset(response); + AuthenticationToken token = Mockito.mock(AuthenticationToken.class); + Mockito.when(token.getUserName()).thenReturn("user"); + Assert.assertFalse(handler.managementOperation(token, request, response)); + Mockito.verify(response).sendError( + Mockito.eq(HttpServletResponse.SC_BAD_REQUEST), + Mockito.contains("requires the parameter [token]")); + + Mockito.reset(response); + StringWriter writer = new StringWriter(); + PrintWriter pwriter = new PrintWriter(writer); + Mockito.when(response.getWriter()).thenReturn(pwriter); + Token dToken = + HttpFSServerWebApp.get().get(DelegationTokenManager.class).createToken( + UserGroupInformation.getCurrentUser(), "user"); + Mockito.when(request.getParameter(HttpFSKerberosAuthenticator.TOKEN_PARAM)). + thenReturn(dToken.encodeToUrlString()); + Assert.assertFalse(handler.managementOperation(token, request, response)); + Mockito.verify(response).setStatus(HttpServletResponse.SC_OK); + pwriter.close(); + Assert.assertTrue(writer.toString().contains("long")); + HttpFSServerWebApp.get().get(DelegationTokenManager.class).verifyToken(dToken); + } + + @Test + @TestDir + public void testAuthenticate() throws Exception { + String dir = TestDirHelper.getTestDir().getAbsolutePath(); + + Configuration httpfsConf = new Configuration(false); + HttpFSServerWebApp server = + new HttpFSServerWebApp(dir, dir, dir, dir, httpfsConf); + server.setAuthority(new InetSocketAddress(InetAddress.getLocalHost(), + 14000)); + AuthenticationHandler handler = + new HttpFSKerberosAuthenticationHandlerForTesting(); + try { + server.init(); + handler.init(null); + + testValidDelegationToken(handler); + testInvalidDelegationToken(handler); + } finally { + if (handler != null) { + handler.destroy(); + } + server.destroy(); + } + } + + private void testValidDelegationToken(AuthenticationHandler handler) + throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Token dToken = + HttpFSServerWebApp.get().get(DelegationTokenManager.class).createToken( + UserGroupInformation.getCurrentUser(), "user"); + Mockito.when(request.getParameter(HttpFSKerberosAuthenticator.DELEGATION_PARAM)). + thenReturn(dToken.encodeToUrlString()); + + AuthenticationToken token = handler.authenticate(request, response); + Assert.assertEquals(UserGroupInformation.getCurrentUser().getShortUserName(), + token.getUserName()); + Assert.assertEquals(0, token.getExpires()); + Assert.assertEquals(HttpFSKerberosAuthenticationHandler.TYPE, + token.getType()); + Assert.assertTrue(token.isExpired()); + } + + private void testInvalidDelegationToken(AuthenticationHandler handler) + throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getParameter(HttpFSKerberosAuthenticator.DELEGATION_PARAM)). + thenReturn("invalid"); + + try { + handler.authenticate(request, response); + Assert.fail(); + } catch (AuthenticationException ex) { + //NOP + } catch (Exception ex) { + Assert.fail(); + } + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java index 099eb4bf81b..910eeefb93e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java @@ -15,11 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.fs.http.server; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.junit.Assert; import java.io.BufferedReader; import java.io.File; @@ -39,9 +37,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator; import org.apache.hadoop.lib.server.Service; import org.apache.hadoop.lib.server.ServiceException; import org.apache.hadoop.lib.service.Groups; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authentication.util.Signer; import org.apache.hadoop.test.HFSTestCase; import org.apache.hadoop.test.HadoopUsersConfTestHelper; import org.apache.hadoop.test.TestDir; @@ -50,6 +52,8 @@ import org.apache.hadoop.test.TestHdfs; import org.apache.hadoop.test.TestHdfsHelper; import org.apache.hadoop.test.TestJetty; import org.apache.hadoop.test.TestJettyHelper; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; import org.junit.Test; import org.mortbay.jetty.Server; import org.mortbay.jetty.webapp.WebAppContext; @@ -103,11 +107,13 @@ public class TestHttpFSServer extends HFSTestCase { } } - private void createHttpFSServer() throws Exception { + + private void createHttpFSServer(boolean addDelegationTokenAuthHandler) + throws Exception { File homeDir = TestDirHelper.getTestDir(); - assertTrue(new File(homeDir, "conf").mkdir()); - assertTrue(new File(homeDir, "log").mkdir()); - assertTrue(new File(homeDir, "temp").mkdir()); + Assert.assertTrue(new File(homeDir, "conf").mkdir()); + Assert.assertTrue(new File(homeDir, "log").mkdir()); + Assert.assertTrue(new File(homeDir, "temp").mkdir()); HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath()); File secretFile = new File(new File(homeDir, "conf"), "secret"); @@ -128,6 +134,10 @@ public class TestHttpFSServer extends HFSTestCase { //HTTPFS configuration conf = new Configuration(false); + if (addDelegationTokenAuthHandler) { + conf.set("httpfs.authentication.type", + HttpFSKerberosAuthenticationHandlerForTesting.class.getName()); + } conf.set("httpfs.services.ext", MockGroups.class.getName()); conf.set("httpfs.admin.group", HadoopUsersConfTestHelper. getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]); @@ -147,6 +157,9 @@ public class TestHttpFSServer extends HFSTestCase { Server server = TestJettyHelper.getJettyServer(); server.addHandler(context); server.start(); + if (addDelegationTokenAuthHandler) { + HttpFSServerWebApp.get().setAuthority(TestJettyHelper.getAuthority()); + } } @Test @@ -154,28 +167,28 @@ public class TestHttpFSServer extends HFSTestCase { @TestJetty @TestHdfs public void instrumentation() throws Exception { - createHttpFSServer(); + createHttpFSServer(false); URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", "nobody")); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_UNAUTHORIZED); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_UNAUTHORIZED); url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", HadoopUsersConfTestHelper.getHadoopUsers()[0])); conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); String line = reader.readLine(); reader.close(); - assertTrue(line.contains("\"counters\":{")); + Assert.assertTrue(line.contains("\"counters\":{")); url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1/foo?user.name={0}&op=instrumentation", HadoopUsersConfTestHelper.getHadoopUsers()[0])); conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); } @Test @@ -183,13 +196,13 @@ public class TestHttpFSServer extends HFSTestCase { @TestJetty @TestHdfs public void testHdfsAccess() throws Exception { - createHttpFSServer(); + createHttpFSServer(false); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1/?user.name={0}&op=liststatus", user)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); reader.readLine(); reader.close(); @@ -200,7 +213,7 @@ public class TestHttpFSServer extends HFSTestCase { @TestJetty @TestHdfs public void testGlobFilter() throws Exception { - createHttpFSServer(); + createHttpFSServer(false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); @@ -210,7 +223,7 @@ public class TestHttpFSServer extends HFSTestCase { URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*", user)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); reader.readLine(); reader.close(); @@ -221,7 +234,7 @@ public class TestHttpFSServer extends HFSTestCase { @TestJetty @TestHdfs public void testPutNoOperation() throws Exception { - createHttpFSServer(); + createHttpFSServer(false); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), @@ -230,7 +243,87 @@ public class TestHttpFSServer extends HFSTestCase { conn.setDoInput(true); conn.setDoOutput(true); conn.setRequestMethod("PUT"); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testDelegationTokenOperations() throws Exception { + createHttpFSServer(true); + + URL url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, + conn.getResponseCode()); + + + AuthenticationToken token = + new AuthenticationToken("u", "p", + HttpFSKerberosAuthenticationHandlerForTesting.TYPE); + token.setExpires(System.currentTimeMillis() + 100000000); + Signer signer = new Signer("secret".getBytes()); + String tokenSigned = signer.sign(token.toString()); + + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY"); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty("Cookie", + AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned); + Assert.assertEquals(HttpURLConnection.HTTP_OK, + conn.getResponseCode()); + + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETDELEGATIONTOKEN"); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty("Cookie", + AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned); + Assert.assertEquals(HttpURLConnection.HTTP_OK, + conn.getResponseCode()); + + JSONObject json = (JSONObject) + new JSONParser().parse(new InputStreamReader(conn.getInputStream())); + json = (JSONObject) + json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON); + String tokenStr = (String) + json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); + + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + Assert.assertEquals(HttpURLConnection.HTTP_OK, + conn.getResponseCode()); + + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, + conn.getResponseCode()); + + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("Cookie", + AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned); + Assert.assertEquals(HttpURLConnection.HTTP_OK, + conn.getResponseCode()); + + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + Assert.assertEquals(HttpURLConnection.HTTP_OK, + conn.getResponseCode()); + + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, + conn.getResponseCode()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig new file mode 100644 index 00000000000..099eb4bf81b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.http.server; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Writer; +import java.net.HttpURLConnection; +import java.net.URL; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.lib.server.Service; +import org.apache.hadoop.lib.server.ServiceException; +import org.apache.hadoop.lib.service.Groups; +import org.apache.hadoop.test.HFSTestCase; +import org.apache.hadoop.test.HadoopUsersConfTestHelper; +import org.apache.hadoop.test.TestDir; +import org.apache.hadoop.test.TestDirHelper; +import org.apache.hadoop.test.TestHdfs; +import org.apache.hadoop.test.TestHdfsHelper; +import org.apache.hadoop.test.TestJetty; +import org.apache.hadoop.test.TestJettyHelper; +import org.junit.Test; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.webapp.WebAppContext; + +public class TestHttpFSServer extends HFSTestCase { + + @Test + @TestDir + @TestJetty + public void server() throws Exception { + String dir = TestDirHelper.getTestDir().getAbsolutePath(); + + Configuration httpfsConf = new Configuration(false); + HttpFSServerWebApp server = new HttpFSServerWebApp(dir, dir, dir, dir, httpfsConf); + server.init(); + server.destroy(); + } + + public static class MockGroups implements Service,Groups { + + @Override + public void init(org.apache.hadoop.lib.server.Server server) throws ServiceException { + } + + @Override + public void postInit() throws ServiceException { + } + + @Override + public void destroy() { + } + + @Override + public Class[] getServiceDependencies() { + return new Class[0]; + } + + @Override + public Class getInterface() { + return Groups.class; + } + + @Override + public void serverStatusChange(org.apache.hadoop.lib.server.Server.Status oldStatus, + org.apache.hadoop.lib.server.Server.Status newStatus) throws ServiceException { + } + + @Override + public List getGroups(String user) throws IOException { + return Arrays.asList(HadoopUsersConfTestHelper.getHadoopUserGroups(user)); + } + + } + private void createHttpFSServer() throws Exception { + File homeDir = TestDirHelper.getTestDir(); + assertTrue(new File(homeDir, "conf").mkdir()); + assertTrue(new File(homeDir, "log").mkdir()); + assertTrue(new File(homeDir, "temp").mkdir()); + HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath()); + + File secretFile = new File(new File(homeDir, "conf"), "secret"); + Writer w = new FileWriter(secretFile); + w.write("secret"); + w.close(); + + //HDFS configuration + File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf"); + hadoopConfDir.mkdirs(); + String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); + Configuration conf = new Configuration(false); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName); + File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml"); + OutputStream os = new FileOutputStream(hdfsSite); + conf.writeXml(os); + os.close(); + + //HTTPFS configuration + conf = new Configuration(false); + conf.set("httpfs.services.ext", MockGroups.class.getName()); + conf.set("httpfs.admin.group", HadoopUsersConfTestHelper. + getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]); + conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups", + HadoopUsersConfTestHelper.getHadoopProxyUserGroups()); + conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts", + HadoopUsersConfTestHelper.getHadoopProxyUserHosts()); + conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath()); + File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml"); + os = new FileOutputStream(httpfsSite); + conf.writeXml(os); + os.close(); + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource("webapp"); + WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs"); + Server server = TestJettyHelper.getJettyServer(); + server.addHandler(context); + server.start(); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void instrumentation() throws Exception { + createHttpFSServer(); + + URL url = new URL(TestJettyHelper.getJettyURL(), + MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", "nobody")); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_UNAUTHORIZED); + + url = new URL(TestJettyHelper.getJettyURL(), + MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", + HadoopUsersConfTestHelper.getHadoopUsers()[0])); + conn = (HttpURLConnection) url.openConnection(); + assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String line = reader.readLine(); + reader.close(); + assertTrue(line.contains("\"counters\":{")); + + url = new URL(TestJettyHelper.getJettyURL(), + MessageFormat.format("/webhdfs/v1/foo?user.name={0}&op=instrumentation", + HadoopUsersConfTestHelper.getHadoopUsers()[0])); + conn = (HttpURLConnection) url.openConnection(); + assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testHdfsAccess() throws Exception { + createHttpFSServer(); + + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + URL url = new URL(TestJettyHelper.getJettyURL(), + MessageFormat.format("/webhdfs/v1/?user.name={0}&op=liststatus", user)); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); + reader.readLine(); + reader.close(); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testGlobFilter() throws Exception { + createHttpFSServer(); + + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + fs.mkdirs(new Path("/tmp")); + fs.create(new Path("/tmp/foo.txt")).close(); + + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + URL url = new URL(TestJettyHelper.getJettyURL(), + MessageFormat.format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*", user)); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); + reader.readLine(); + reader.close(); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testPutNoOperation() throws Exception { + createHttpFSServer(); + + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + URL url = new URL(TestJettyHelper.getJettyURL(), + MessageFormat.format("/webhdfs/v1/foo?user.name={0}", user)); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoInput(true); + conn.setDoOutput(true); + conn.setRequestMethod("PUT"); + assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java new file mode 100644 index 00000000000..87fdb3dccdf --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.http.server; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.DelegationTokenRenewer; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.http.client.HttpFSFileSystem; +import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.HFSTestCase; +import org.apache.hadoop.test.KerberosTestUtils; +import org.apache.hadoop.test.TestDir; +import org.apache.hadoop.test.TestDirHelper; +import org.apache.hadoop.test.TestHdfs; +import org.apache.hadoop.test.TestHdfsHelper; +import org.apache.hadoop.test.TestJetty; +import org.apache.hadoop.test.TestJettyHelper; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.After; +import org.junit.Test; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.webapp.WebAppContext; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Writer; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.security.PrivilegedExceptionAction; +import java.util.concurrent.Callable; + +public class TestHttpFSWithKerberos extends HFSTestCase { + + @After + public void resetUGI() { + Configuration conf = new Configuration(); + UserGroupInformation.setConfiguration(conf); + } + + private void createHttpFSServer() throws Exception { + File homeDir = TestDirHelper.getTestDir(); + Assert.assertTrue(new File(homeDir, "conf").mkdir()); + Assert.assertTrue(new File(homeDir, "log").mkdir()); + Assert.assertTrue(new File(homeDir, "temp").mkdir()); + HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath()); + + File secretFile = new File(new File(homeDir, "conf"), "secret"); + Writer w = new FileWriter(secretFile); + w.write("secret"); + w.close(); + + //HDFS configuration + File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf"); + hadoopConfDir.mkdirs(); + String fsDefaultName = TestHdfsHelper.getHdfsConf() + .get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); + Configuration conf = new Configuration(false); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName); + File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml"); + OutputStream os = new FileOutputStream(hdfsSite); + conf.writeXml(os); + os.close(); + + conf = new Configuration(false); + conf.set("httpfs.proxyuser.client.hosts", "*"); + conf.set("httpfs.proxyuser.client.groups", "*"); + + conf.set("httpfs.authentication.type", "kerberos"); + + conf.set("httpfs.authentication.signature.secret.file", + secretFile.getAbsolutePath()); + File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml"); + os = new FileOutputStream(httpfsSite); + conf.writeXml(os); + os.close(); + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource("webapp"); + WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs"); + Server server = TestJettyHelper.getJettyServer(); + server.addHandler(context); + server.start(); + HttpFSServerWebApp.get().setAuthority(TestJettyHelper.getAuthority()); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testValidHttpFSAccess() throws Exception { + createHttpFSServer(); + + KerberosTestUtils.doAsClient(new Callable() { + @Override + public Void call() throws Exception { + URL url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY"); + AuthenticatedURL aUrl = new AuthenticatedURL(); + AuthenticatedURL.Token aToken = new AuthenticatedURL.Token(); + HttpURLConnection conn = aUrl.openConnection(url, aToken); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + return null; + } + }); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testInvalidadHttpFSAccess() throws Exception { + createHttpFSServer(); + + URL url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + Assert.assertEquals(conn.getResponseCode(), + HttpURLConnection.HTTP_UNAUTHORIZED); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testDelegationTokenHttpFSAccess() throws Exception { + createHttpFSServer(); + + KerberosTestUtils.doAsClient(new Callable() { + @Override + public Void call() throws Exception { + //get delegation token doing SPNEGO authentication + URL url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETDELEGATIONTOKEN"); + AuthenticatedURL aUrl = new AuthenticatedURL(); + AuthenticatedURL.Token aToken = new AuthenticatedURL.Token(); + HttpURLConnection conn = aUrl.openConnection(url, aToken); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) new JSONParser() + .parse(new InputStreamReader(conn.getInputStream())); + json = + (JSONObject) json + .get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON); + String tokenStr = (String) json + .get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); + + //access httpfs using the delegation token + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + + //try to renew the delegation token without SPNEGO credentials + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + Assert.assertEquals(conn.getResponseCode(), + HttpURLConnection.HTTP_UNAUTHORIZED); + + //renew the delegation token with SPNEGO credentials + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr); + conn = aUrl.openConnection(url, aToken); + conn.setRequestMethod("PUT"); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + + //cancel delegation token, no need for SPNEGO credentials + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + + //try to access httpfs with the canceled delegation token + url = new URL(TestJettyHelper.getJettyURL(), + "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + + tokenStr); + conn = (HttpURLConnection) url.openConnection(); + Assert.assertEquals(conn.getResponseCode(), + HttpURLConnection.HTTP_UNAUTHORIZED); + return null; + } + }); + } + + @SuppressWarnings("deprecation") + private void testDelegationTokenWithFS(Class fileSystemClass) + throws Exception { + createHttpFSServer(); + Configuration conf = new Configuration(); + conf.set("fs.webhdfs.impl", fileSystemClass.getName()); + conf.set("fs.hdfs.impl.disable.cache", "true"); + URI uri = new URI( "webhdfs://" + + TestJettyHelper.getJettyURL().toURI().getAuthority()); + FileSystem fs = FileSystem.get(uri, conf); + Token token = fs.getDelegationToken("foo"); + fs.close(); + fs = FileSystem.get(uri, conf); + ((DelegationTokenRenewer.Renewable) fs).setDelegationToken(token); + fs.listStatus(new Path("/")); + fs.close(); + } + + private void testDelegationTokenWithinDoAs( + final Class fileSystemClass, boolean proxyUser) throws Exception { + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(conf); + UserGroupInformation.loginUserFromKeytab("client", + "/Users/tucu/tucu.keytab"); + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + if (proxyUser) { + ugi = UserGroupInformation.createProxyUser("foo", ugi); + } + conf = new Configuration(); + UserGroupInformation.setConfiguration(conf); + ugi.doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + testDelegationTokenWithFS(fileSystemClass); + return null; + } + }); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testDelegationTokenWithHttpFSFileSystem() throws Exception { + testDelegationTokenWithinDoAs(HttpFSFileSystem.class, false); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testDelegationTokenWithWebhdfsFileSystem() throws Exception { + testDelegationTokenWithinDoAs(WebHdfsFileSystem.class, false); + } + + @Test + @TestDir + @TestJetty + @TestHdfs + public void testDelegationTokenWithHttpFSFileSystemProxyUser() + throws Exception { + testDelegationTokenWithinDoAs(HttpFSFileSystem.class, true); + } + + // TODO: WebHdfsFilesystem does work with ProxyUser HDFS-3509 + // @Test + // @TestDir + // @TestJetty + // @TestHdfs + // public void testDelegationTokenWithWebhdfsFileSystemProxyUser() + // throws Exception { + // testDelegationTokenWithinDoAs(WebHdfsFileSystem.class, true); + // } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestDelegationTokenManagerService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestDelegationTokenManagerService.java new file mode 100644 index 00000000000..480d04c8f42 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestDelegationTokenManagerService.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.lib.service.security; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; +import org.apache.hadoop.lib.server.Server; +import org.apache.hadoop.lib.service.DelegationTokenManager; +import org.apache.hadoop.lib.service.DelegationTokenManagerException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.HTestCase; +import org.apache.hadoop.test.TestDir; +import org.apache.hadoop.test.TestDirHelper; +import org.apache.hadoop.util.StringUtils; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Arrays; + +public class TestDelegationTokenManagerService extends HTestCase { + + @Test + @TestDir + public void service() throws Exception { + String dir = TestDirHelper.getTestDir().getAbsolutePath(); + Configuration conf = new Configuration(false); + conf.set("server.services", StringUtils.join(",", + Arrays.asList(DelegationTokenManagerService.class.getName()))); + Server server = new Server("server", dir, dir, dir, dir, conf); + server.init(); + DelegationTokenManager tm = server.get(DelegationTokenManager.class); + Assert.assertNotNull(tm); + server.destroy(); + } + + @Test + @TestDir + @SuppressWarnings("unchecked") + public void tokens() throws Exception { + String dir = TestDirHelper.getTestDir().getAbsolutePath(); + Configuration conf = new Configuration(false); + conf.set("server.services", StringUtils.join(",", + Arrays.asList(DelegationTokenManagerService.class.getName()))); + HttpFSServerWebApp server = new HttpFSServerWebApp(dir, dir, dir, dir, conf); + server.setAuthority(new InetSocketAddress(InetAddress.getLocalHost(), 14000)); + server.init(); + DelegationTokenManager tm = server.get(DelegationTokenManager.class); + Token token = tm.createToken(UserGroupInformation.getCurrentUser(), "foo"); + Assert.assertNotNull(token); + tm.verifyToken(token); + Assert.assertTrue(tm.renewToken(token, "foo") > System.currentTimeMillis()); + tm.cancelToken(token, "foo"); + try { + tm.verifyToken(token); + Assert.fail(); + } catch (DelegationTokenManagerException ex) { + //NOP + } catch (Exception ex) { + Assert.fail(); + } + server.destroy(); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/KerberosTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/KerberosTestUtils.java new file mode 100644 index 00000000000..4d289ad1ddd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/KerberosTestUtils.java @@ -0,0 +1,138 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.test; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; + +import org.apache.hadoop.security.authentication.util.KerberosUtil; + +import java.io.File; +import java.security.Principal; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; + +/** + * Test helper class for Java Kerberos setup. + */ +public class KerberosTestUtils { + private static final String PREFIX = "httpfs.test."; + + public static final String REALM = PREFIX + "kerberos.realm"; + + public static final String CLIENT_PRINCIPAL = + PREFIX + "kerberos.client.principal"; + + public static final String SERVER_PRINCIPAL = + PREFIX + "kerberos.server.principal"; + + public static final String KEYTAB_FILE = PREFIX + "kerberos.keytab.file"; + + public static String getRealm() { + return System.getProperty(REALM, "LOCALHOST"); + } + + public static String getClientPrincipal() { + return System.getProperty(CLIENT_PRINCIPAL, "client") + "@" + getRealm(); + } + + public static String getServerPrincipal() { + return System.getProperty(SERVER_PRINCIPAL, + "HTTP/localhost") + "@" + getRealm(); + } + + public static String getKeytabFile() { + String keytabFile = + new File(System.getProperty("user.home"), + System.getProperty("user.name") + ".keytab").toString(); + return System.getProperty(KEYTAB_FILE, keytabFile); + } + + private static class KerberosConfiguration extends Configuration { + private String principal; + + public KerberosConfiguration(String principal) { + this.principal = principal; + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) { + Map options = new HashMap(); + options.put("keyTab", KerberosTestUtils.getKeytabFile()); + options.put("principal", principal); + options.put("useKeyTab", "true"); + options.put("storeKey", "true"); + options.put("doNotPrompt", "true"); + options.put("useTicketCache", "true"); + options.put("renewTGT", "true"); + options.put("refreshKrb5Config", "true"); + options.put("isInitiator", "true"); + String ticketCache = System.getenv("KRB5CCNAME"); + if (ticketCache != null) { + options.put("ticketCache", ticketCache); + } + options.put("debug", "true"); + + return new AppConfigurationEntry[]{ + new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(), + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, + options),}; + } + } + + public static T doAs(String principal, final Callable callable) + throws Exception { + LoginContext loginContext = null; + try { + Set principals = new HashSet(); + principals.add( + new KerberosPrincipal(KerberosTestUtils.getClientPrincipal())); + Subject subject = new Subject(false, principals, new HashSet(), + new HashSet()); + loginContext = new LoginContext("", subject, null, + new KerberosConfiguration(principal)); + loginContext.login(); + subject = loginContext.getSubject(); + return Subject.doAs(subject, new PrivilegedExceptionAction() { + @Override + public T run() throws Exception { + return callable.call(); + } + }); + } catch (PrivilegedActionException ex) { + throw ex.getException(); + } finally { + if (loginContext != null) { + loginContext.logout(); + } + } + } + + public static T doAsClient(Callable callable) throws Exception { + return doAs(getClientPrincipal(), callable); + } + + public static T doAsServer(Callable callable) throws Exception { + return doAs(getServerPrincipal(), callable); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java index 3368c79c7a2..4d11691c67e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java @@ -73,7 +73,7 @@ public class TestDirHelper implements MethodRule { System.exit(-1); } - TEST_DIR_ROOT = new File(TEST_DIR_ROOT, "testdir").getAbsolutePath(); + TEST_DIR_ROOT = new File(TEST_DIR_ROOT, "test-dir").getAbsolutePath(); System.setProperty(TEST_DIR_PROP, TEST_DIR_ROOT); File dir = new File(TEST_DIR_ROOT); @@ -83,8 +83,6 @@ public class TestDirHelper implements MethodRule { System.exit(-1); } - System.setProperty("test.circus", "true"); - System.out.println(">>> " + TEST_DIR_PROP + " : " + System.getProperty(TEST_DIR_PROP)); } catch (IOException ex) { throw new RuntimeException(ex); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java index 95cb10463cd..4442281fd84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java @@ -18,9 +18,11 @@ package org.apache.hadoop.test; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.ServerSocket; import java.net.URL; +import java.net.UnknownHostException; import org.junit.Test; import org.junit.rules.MethodRule; @@ -65,9 +67,9 @@ public class TestJettyHelper implements MethodRule { private Server createJettyServer() { try { - - String host = InetAddress.getLocalHost().getHostName(); - ServerSocket ss = new ServerSocket(0); + InetAddress localhost = InetAddress.getByName("localhost"); + String host = "localhost"; + ServerSocket ss = new ServerSocket(0, 50, localhost); int port = ss.getLocalPort(); ss.close(); Server server = new Server(0); @@ -79,6 +81,23 @@ public class TestJettyHelper implements MethodRule { } } + /** + * Returns the authority (hostname & port) used by the JettyServer. + * + * @return an InetSocketAddress with the corresponding authority. + */ + public static InetSocketAddress getAuthority() { + Server server = getJettyServer(); + try { + InetAddress add = + InetAddress.getByName(server.getConnectors()[0].getHost()); + int port = server.getConnectors()[0].getPort(); + return new InetSocketAddress(add, port); + } catch (UnknownHostException ex) { + throw new RuntimeException(ex); + } + } + /** * Returns a Jetty server ready to be configured and the started. This server * is only available when the test method has been annotated with diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig new file mode 100644 index 00000000000..95cb10463cd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.test; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.ServerSocket; +import java.net.URL; + +import org.junit.Test; +import org.junit.rules.MethodRule; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.Statement; +import org.mortbay.jetty.Server; + +public class TestJettyHelper implements MethodRule { + + @Test + public void dummy() { + } + + private static ThreadLocal TEST_SERVLET_TL = new InheritableThreadLocal(); + + @Override + public Statement apply(final Statement statement, final FrameworkMethod frameworkMethod, final Object o) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + Server server = null; + TestJetty testJetty = frameworkMethod.getAnnotation(TestJetty.class); + if (testJetty != null) { + server = createJettyServer(); + } + try { + TEST_SERVLET_TL.set(server); + statement.evaluate(); + } finally { + TEST_SERVLET_TL.remove(); + if (server != null && server.isRunning()) { + try { + server.stop(); + } catch (Exception ex) { + throw new RuntimeException("Could not stop embedded servlet container, " + ex.getMessage(), ex); + } + } + } + } + }; + } + + private Server createJettyServer() { + try { + + String host = InetAddress.getLocalHost().getHostName(); + ServerSocket ss = new ServerSocket(0); + int port = ss.getLocalPort(); + ss.close(); + Server server = new Server(0); + server.getConnectors()[0].setHost(host); + server.getConnectors()[0].setPort(port); + return server; + } catch (Exception ex) { + throw new RuntimeException("Could not stop embedded servlet container, " + ex.getMessage(), ex); + } + } + + /** + * Returns a Jetty server ready to be configured and the started. This server + * is only available when the test method has been annotated with + * {@link TestJetty}. Refer to {@link HTestCase} header for details. + *

+ * Once configured, the Jetty server should be started. The server will be + * automatically stopped when the test method ends. + * + * @return a Jetty server ready to be configured and the started. + */ + public static Server getJettyServer() { + Server server = TEST_SERVLET_TL.get(); + if (server == null) { + throw new IllegalStateException("This test does not use @TestJetty"); + } + return server; + } + + /** + * Returns the base URL (SCHEMA://HOST:PORT) of the test Jetty server + * (see {@link #getJettyServer()}) once started. + * + * @return the base URL (SCHEMA://HOST:PORT) of the test Jetty server. + */ + public static URL getJettyURL() { + Server server = TEST_SERVLET_TL.get(); + if (server == null) { + throw new IllegalStateException("This test does not use @TestJetty"); + } + try { + return new URL("http://" + server.getConnectors()[0].getHost() + ":" + server.getConnectors()[0].getPort()); + } catch (MalformedURLException ex) { + throw new RuntimeException("It should never happen, " + ex.getMessage(), ex); + } + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/krb5.conf b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/krb5.conf new file mode 100644 index 00000000000..60dc2961201 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/krb5.conf @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[libdefaults] + default_realm = ${kerberos.realm} + udp_preference_limit = 1 + extra_addresses = 127.0.0.1 +[realms] + ${kerberos.realm} = { + admin_server = localhost:88 + kdc = localhost:88 + } +[domain_realm] + localhost = ${kerberos.realm} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5d1e01ef65d..d90dda13578 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -199,6 +199,8 @@ Branch-2 ( Unreleased changes ) HDFS-3518. Add a utility method HdfsUtils.isHealthy(uri) for checking if the given HDFS is healthy. (szetszwo) + HDFS-3113. httpfs does not support delegation tokens. (tucu) + IMPROVEMENTS HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG From c5fe08e0ea470afdb8ba93c10f71530b41694e19 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 26 Jul 2012 13:42:26 +0000 Subject: [PATCH 22/39] HDFS-3113 amendment, removing incorrectly committed files git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1365990 13f79535-47bb-0310-9956-ffa450edef68 --- .../client/TestHttpFSFileSystem.java.orig | 513 ------------------ .../fs/http/server/TestHttpFSServer.java.orig | 236 -------- .../hadoop/test/TestJettyHelper.java.orig | 118 ---- 3 files changed, 867 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig deleted file mode 100644 index ebebd25003f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java.orig +++ /dev/null @@ -1,513 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.http.client; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Writer; -import java.net.URL; -import java.security.PrivilegedExceptionAction; -import java.util.Arrays; -import java.util.Collection; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.fs.FileChecksum; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.test.HFSTestCase; -import org.apache.hadoop.test.HadoopUsersConfTestHelper; -import org.apache.hadoop.test.TestDir; -import org.apache.hadoop.test.TestDirHelper; -import org.apache.hadoop.test.TestHdfs; -import org.apache.hadoop.test.TestHdfsHelper; -import org.apache.hadoop.test.TestJetty; -import org.apache.hadoop.test.TestJettyHelper; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.webapp.WebAppContext; - -@RunWith(value = Parameterized.class) -public class TestHttpFSFileSystem extends HFSTestCase { - - private void createHttpFSServer() throws Exception { - File homeDir = TestDirHelper.getTestDir(); - Assert.assertTrue(new File(homeDir, "conf").mkdir()); - Assert.assertTrue(new File(homeDir, "log").mkdir()); - Assert.assertTrue(new File(homeDir, "temp").mkdir()); - HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath()); - - File secretFile = new File(new File(homeDir, "conf"), "secret"); - Writer w = new FileWriter(secretFile); - w.write("secret"); - w.close(); - - //HDFS configuration - String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); - Configuration conf = new Configuration(false); - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName); - File hdfsSite = new File(new File(homeDir, "conf"), "hdfs-site.xml"); - OutputStream os = new FileOutputStream(hdfsSite); - conf.writeXml(os); - os.close(); - - //HTTPFS configuration - conf = new Configuration(false); - conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups", - HadoopUsersConfTestHelper.getHadoopProxyUserGroups()); - conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts", - HadoopUsersConfTestHelper.getHadoopProxyUserHosts()); - conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath()); - File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml"); - os = new FileOutputStream(httpfsSite); - conf.writeXml(os); - os.close(); - - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - URL url = cl.getResource("webapp"); - WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs"); - Server server = TestJettyHelper.getJettyServer(); - server.addHandler(context); - server.start(); - } - - protected FileSystem getHttpFileSystem() throws Exception { - Configuration conf = new Configuration(); - conf.set("fs.http.impl", HttpFSFileSystem.class.getName()); - return FileSystem.get(TestJettyHelper.getJettyURL().toURI(), conf); - } - - protected void testGet() throws Exception { - FileSystem fs = getHttpFileSystem(); - Assert.assertNotNull(fs); - Assert.assertEquals(fs.getUri(), TestJettyHelper.getJettyURL().toURI()); - fs.close(); - } - - private void testOpen() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - fs.close(); - fs = getHttpFileSystem(); - InputStream is = fs.open(new Path(path.toUri().getPath())); - Assert.assertEquals(is.read(), 1); - is.close(); - fs.close(); - } - - private void testCreate(Path path, boolean override) throws Exception { - FileSystem fs = getHttpFileSystem(); - FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE); - OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024, - (short) 2, 100 * 1024 * 1024, null); - os.write(1); - os.close(); - fs.close(); - - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - FileStatus status = fs.getFileStatus(path); - Assert.assertEquals(status.getReplication(), 2); - Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024); - Assert.assertEquals(status.getPermission(), permission); - InputStream is = fs.open(path); - Assert.assertEquals(is.read(), 1); - is.close(); - fs.close(); - } - - private void testCreate() throws Exception { - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - testCreate(path, false); - testCreate(path, true); - try { - testCreate(path, false); - Assert.fail(); - } catch (IOException ex) { - - } catch (Exception ex) { - Assert.fail(); - } - } - - private void testAppend() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - fs.close(); - fs = getHttpFileSystem(); - os = fs.append(new Path(path.toUri().getPath())); - os.write(2); - os.close(); - fs.close(); - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - InputStream is = fs.open(path); - Assert.assertEquals(is.read(), 1); - Assert.assertEquals(is.read(), 2); - Assert.assertEquals(is.read(), -1); - is.close(); - fs.close(); - } - - private void testRename() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo"); - fs.mkdirs(path); - fs.close(); - fs = getHttpFileSystem(); - Path oldPath = new Path(path.toUri().getPath()); - Path newPath = new Path(path.getParent(), "bar"); - fs.rename(oldPath, newPath); - fs.close(); - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Assert.assertFalse(fs.exists(oldPath)); - Assert.assertTrue(fs.exists(newPath)); - fs.close(); - } - - private void testDelete() throws Exception { - Path foo = new Path(TestHdfsHelper.getHdfsTestDir(), "foo"); - Path bar = new Path(TestHdfsHelper.getHdfsTestDir(), "bar"); - Path foe = new Path(TestHdfsHelper.getHdfsTestDir(), "foe"); - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - fs.mkdirs(foo); - fs.mkdirs(new Path(bar, "a")); - fs.mkdirs(foe); - - FileSystem hoopFs = getHttpFileSystem(); - Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false)); - Assert.assertFalse(fs.exists(foo)); - try { - hoopFs.delete(new Path(bar.toUri().getPath()), false); - Assert.fail(); - } catch (IOException ex) { - } catch (Exception ex) { - Assert.fail(); - } - Assert.assertTrue(fs.exists(bar)); - Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true)); - Assert.assertFalse(fs.exists(bar)); - - Assert.assertTrue(fs.exists(foe)); - Assert.assertTrue(hoopFs.delete(foe, true)); - Assert.assertFalse(fs.exists(foe)); - - hoopFs.close(); - fs.close(); - } - - private void testListStatus() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - FileStatus status1 = fs.getFileStatus(path); - fs.close(); - - fs = getHttpFileSystem(); - FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath())); - fs.close(); - - Assert.assertEquals(status2.getPermission(), status1.getPermission()); - Assert.assertEquals(status2.getPath().toUri().getPath(), status1.getPath().toUri().getPath()); - Assert.assertEquals(status2.getReplication(), status1.getReplication()); - Assert.assertEquals(status2.getBlockSize(), status1.getBlockSize()); - Assert.assertEquals(status2.getAccessTime(), status1.getAccessTime()); - Assert.assertEquals(status2.getModificationTime(), status1.getModificationTime()); - Assert.assertEquals(status2.getOwner(), status1.getOwner()); - Assert.assertEquals(status2.getGroup(), status1.getGroup()); - Assert.assertEquals(status2.getLen(), status1.getLen()); - - FileStatus[] stati = fs.listStatus(path.getParent()); - Assert.assertEquals(stati.length, 1); - Assert.assertEquals(stati[0].getPath().getName(), path.getName()); - } - - private void testWorkingdirectory() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path workingDir = fs.getWorkingDirectory(); - fs.close(); - - fs = getHttpFileSystem(); - Path hoopWorkingDir = fs.getWorkingDirectory(); - fs.close(); - Assert.assertEquals(hoopWorkingDir.toUri().getPath(), workingDir.toUri().getPath()); - - fs = getHttpFileSystem(); - fs.setWorkingDirectory(new Path("/tmp")); - workingDir = fs.getWorkingDirectory(); - fs.close(); - Assert.assertEquals(workingDir.toUri().getPath(), new Path("/tmp").toUri().getPath()); - } - - private void testMkdirs() throws Exception { - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo"); - FileSystem fs = getHttpFileSystem(); - fs.mkdirs(path); - fs.close(); - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Assert.assertTrue(fs.exists(path)); - fs.close(); - } - - private void testSetTimes() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - FileStatus status1 = fs.getFileStatus(path); - fs.close(); - long at = status1.getAccessTime(); - long mt = status1.getModificationTime(); - - fs = getHttpFileSystem(); - fs.setTimes(path, mt + 10, at + 20); - fs.close(); - - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - status1 = fs.getFileStatus(path); - fs.close(); - long atNew = status1.getAccessTime(); - long mtNew = status1.getModificationTime(); - Assert.assertEquals(mtNew, mt + 10); - Assert.assertEquals(atNew, at + 20); - } - - private void testSetPermission() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foodir"); - fs.mkdirs(path); - - fs = getHttpFileSystem(); - FsPermission permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE); - fs.setPermission(path, permission1); - fs.close(); - - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - FileStatus status1 = fs.getFileStatus(path); - fs.close(); - FsPermission permission2 = status1.getPermission(); - Assert.assertEquals(permission2, permission1); - - //sticky bit - fs = getHttpFileSystem(); - permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE, true); - fs.setPermission(path, permission1); - fs.close(); - - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - status1 = fs.getFileStatus(path); - fs.close(); - permission2 = status1.getPermission(); - Assert.assertTrue(permission2.getStickyBit()); - Assert.assertEquals(permission2, permission1); - } - - private void testSetOwner() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - fs.close(); - - fs = getHttpFileSystem(); - String user = HadoopUsersConfTestHelper.getHadoopUsers()[1]; - String group = HadoopUsersConfTestHelper.getHadoopUserGroups(user)[0]; - fs.setOwner(path, user, group); - fs.close(); - - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - FileStatus status1 = fs.getFileStatus(path); - fs.close(); - Assert.assertEquals(status1.getOwner(), user); - Assert.assertEquals(status1.getGroup(), group); - } - - private void testSetReplication() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - fs.close(); - fs.setReplication(path, (short) 2); - - fs = getHttpFileSystem(); - fs.setReplication(path, (short) 1); - fs.close(); - - fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - FileStatus status1 = fs.getFileStatus(path); - fs.close(); - Assert.assertEquals(status1.getReplication(), (short) 1); - } - - private void testChecksum() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - FileChecksum hdfsChecksum = fs.getFileChecksum(path); - fs.close(); - fs = getHttpFileSystem(); - FileChecksum httpChecksum = fs.getFileChecksum(path); - fs.close(); - Assert.assertEquals(httpChecksum.getAlgorithmName(), hdfsChecksum.getAlgorithmName()); - Assert.assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength()); - Assert.assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes()); - } - - private void testContentSummary() throws Exception { - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt"); - OutputStream os = fs.create(path); - os.write(1); - os.close(); - ContentSummary hdfsContentSummary = fs.getContentSummary(path); - fs.close(); - fs = getHttpFileSystem(); - ContentSummary httpContentSummary = fs.getContentSummary(path); - fs.close(); - Assert.assertEquals(httpContentSummary.getDirectoryCount(), hdfsContentSummary.getDirectoryCount()); - Assert.assertEquals(httpContentSummary.getFileCount(), hdfsContentSummary.getFileCount()); - Assert.assertEquals(httpContentSummary.getLength(), hdfsContentSummary.getLength()); - Assert.assertEquals(httpContentSummary.getQuota(), hdfsContentSummary.getQuota()); - Assert.assertEquals(httpContentSummary.getSpaceConsumed(), hdfsContentSummary.getSpaceConsumed()); - Assert.assertEquals(httpContentSummary.getSpaceQuota(), hdfsContentSummary.getSpaceQuota()); - } - - protected enum Operation { - GET, OPEN, CREATE, APPEND, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, - SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY - } - - private void operation(Operation op) throws Exception { - switch (op) { - case GET: - testGet(); - break; - case OPEN: - testOpen(); - break; - case CREATE: - testCreate(); - break; - case APPEND: - testAppend(); - break; - case RENAME: - testRename(); - break; - case DELETE: - testDelete(); - break; - case LIST_STATUS: - testListStatus(); - break; - case WORKING_DIRECTORY: - testWorkingdirectory(); - break; - case MKDIRS: - testMkdirs(); - break; - case SET_TIMES: - testSetTimes(); - break; - case SET_PERMISSION: - testSetPermission(); - break; - case SET_OWNER: - testSetOwner(); - break; - case SET_REPLICATION: - testSetReplication(); - break; - case CHECKSUM: - testChecksum(); - break; - case CONTENT_SUMMARY: - testContentSummary(); - break; - } - } - - @Parameterized.Parameters - public static Collection operations() { - Object[][] ops = new Object[Operation.values().length][]; - for (int i = 0; i < Operation.values().length; i++) { - ops[i] = new Object[]{Operation.values()[i]}; - } - return Arrays.asList(ops); -// return Arrays.asList(new Object[][]{ new Object[]{Operation.CREATE}}); - } - - private Operation operation; - - public TestHttpFSFileSystem(Operation operation) { - this.operation = operation; - } - - @Test - @TestDir - @TestJetty - @TestHdfs - public void testOperation() throws Exception { - createHttpFSServer(); - operation(operation); - } - - @Test - @TestDir - @TestJetty - @TestHdfs - public void testOperationDoAs() throws Exception { - createHttpFSServer(); - UserGroupInformation ugi = UserGroupInformation.createProxyUser(HadoopUsersConfTestHelper.getHadoopUsers()[0], - UserGroupInformation.getCurrentUser()); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - operation(operation); - return null; - } - }); - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig deleted file mode 100644 index 099eb4bf81b..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java.orig +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.http.server; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.Writer; -import java.net.HttpURLConnection; -import java.net.URL; -import java.text.MessageFormat; -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.lib.server.Service; -import org.apache.hadoop.lib.server.ServiceException; -import org.apache.hadoop.lib.service.Groups; -import org.apache.hadoop.test.HFSTestCase; -import org.apache.hadoop.test.HadoopUsersConfTestHelper; -import org.apache.hadoop.test.TestDir; -import org.apache.hadoop.test.TestDirHelper; -import org.apache.hadoop.test.TestHdfs; -import org.apache.hadoop.test.TestHdfsHelper; -import org.apache.hadoop.test.TestJetty; -import org.apache.hadoop.test.TestJettyHelper; -import org.junit.Test; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.webapp.WebAppContext; - -public class TestHttpFSServer extends HFSTestCase { - - @Test - @TestDir - @TestJetty - public void server() throws Exception { - String dir = TestDirHelper.getTestDir().getAbsolutePath(); - - Configuration httpfsConf = new Configuration(false); - HttpFSServerWebApp server = new HttpFSServerWebApp(dir, dir, dir, dir, httpfsConf); - server.init(); - server.destroy(); - } - - public static class MockGroups implements Service,Groups { - - @Override - public void init(org.apache.hadoop.lib.server.Server server) throws ServiceException { - } - - @Override - public void postInit() throws ServiceException { - } - - @Override - public void destroy() { - } - - @Override - public Class[] getServiceDependencies() { - return new Class[0]; - } - - @Override - public Class getInterface() { - return Groups.class; - } - - @Override - public void serverStatusChange(org.apache.hadoop.lib.server.Server.Status oldStatus, - org.apache.hadoop.lib.server.Server.Status newStatus) throws ServiceException { - } - - @Override - public List getGroups(String user) throws IOException { - return Arrays.asList(HadoopUsersConfTestHelper.getHadoopUserGroups(user)); - } - - } - private void createHttpFSServer() throws Exception { - File homeDir = TestDirHelper.getTestDir(); - assertTrue(new File(homeDir, "conf").mkdir()); - assertTrue(new File(homeDir, "log").mkdir()); - assertTrue(new File(homeDir, "temp").mkdir()); - HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath()); - - File secretFile = new File(new File(homeDir, "conf"), "secret"); - Writer w = new FileWriter(secretFile); - w.write("secret"); - w.close(); - - //HDFS configuration - File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf"); - hadoopConfDir.mkdirs(); - String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); - Configuration conf = new Configuration(false); - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName); - File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml"); - OutputStream os = new FileOutputStream(hdfsSite); - conf.writeXml(os); - os.close(); - - //HTTPFS configuration - conf = new Configuration(false); - conf.set("httpfs.services.ext", MockGroups.class.getName()); - conf.set("httpfs.admin.group", HadoopUsersConfTestHelper. - getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]); - conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups", - HadoopUsersConfTestHelper.getHadoopProxyUserGroups()); - conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts", - HadoopUsersConfTestHelper.getHadoopProxyUserHosts()); - conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath()); - File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml"); - os = new FileOutputStream(httpfsSite); - conf.writeXml(os); - os.close(); - - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - URL url = cl.getResource("webapp"); - WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs"); - Server server = TestJettyHelper.getJettyServer(); - server.addHandler(context); - server.start(); - } - - @Test - @TestDir - @TestJetty - @TestHdfs - public void instrumentation() throws Exception { - createHttpFSServer(); - - URL url = new URL(TestJettyHelper.getJettyURL(), - MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", "nobody")); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_UNAUTHORIZED); - - url = new URL(TestJettyHelper.getJettyURL(), - MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", - HadoopUsersConfTestHelper.getHadoopUsers()[0])); - conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); - BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); - String line = reader.readLine(); - reader.close(); - assertTrue(line.contains("\"counters\":{")); - - url = new URL(TestJettyHelper.getJettyURL(), - MessageFormat.format("/webhdfs/v1/foo?user.name={0}&op=instrumentation", - HadoopUsersConfTestHelper.getHadoopUsers()[0])); - conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); - } - - @Test - @TestDir - @TestJetty - @TestHdfs - public void testHdfsAccess() throws Exception { - createHttpFSServer(); - - String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; - URL url = new URL(TestJettyHelper.getJettyURL(), - MessageFormat.format("/webhdfs/v1/?user.name={0}&op=liststatus", user)); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); - BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); - reader.readLine(); - reader.close(); - } - - @Test - @TestDir - @TestJetty - @TestHdfs - public void testGlobFilter() throws Exception { - createHttpFSServer(); - - FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); - fs.mkdirs(new Path("/tmp")); - fs.create(new Path("/tmp/foo.txt")).close(); - - String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; - URL url = new URL(TestJettyHelper.getJettyURL(), - MessageFormat.format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*", user)); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); - BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); - reader.readLine(); - reader.close(); - } - - @Test - @TestDir - @TestJetty - @TestHdfs - public void testPutNoOperation() throws Exception { - createHttpFSServer(); - - String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; - URL url = new URL(TestJettyHelper.getJettyURL(), - MessageFormat.format("/webhdfs/v1/foo?user.name={0}", user)); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setDoInput(true); - conn.setDoOutput(true); - conn.setRequestMethod("PUT"); - assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST); - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig deleted file mode 100644 index 95cb10463cd..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java.orig +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.test; - -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.ServerSocket; -import java.net.URL; - -import org.junit.Test; -import org.junit.rules.MethodRule; -import org.junit.runners.model.FrameworkMethod; -import org.junit.runners.model.Statement; -import org.mortbay.jetty.Server; - -public class TestJettyHelper implements MethodRule { - - @Test - public void dummy() { - } - - private static ThreadLocal TEST_SERVLET_TL = new InheritableThreadLocal(); - - @Override - public Statement apply(final Statement statement, final FrameworkMethod frameworkMethod, final Object o) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - Server server = null; - TestJetty testJetty = frameworkMethod.getAnnotation(TestJetty.class); - if (testJetty != null) { - server = createJettyServer(); - } - try { - TEST_SERVLET_TL.set(server); - statement.evaluate(); - } finally { - TEST_SERVLET_TL.remove(); - if (server != null && server.isRunning()) { - try { - server.stop(); - } catch (Exception ex) { - throw new RuntimeException("Could not stop embedded servlet container, " + ex.getMessage(), ex); - } - } - } - } - }; - } - - private Server createJettyServer() { - try { - - String host = InetAddress.getLocalHost().getHostName(); - ServerSocket ss = new ServerSocket(0); - int port = ss.getLocalPort(); - ss.close(); - Server server = new Server(0); - server.getConnectors()[0].setHost(host); - server.getConnectors()[0].setPort(port); - return server; - } catch (Exception ex) { - throw new RuntimeException("Could not stop embedded servlet container, " + ex.getMessage(), ex); - } - } - - /** - * Returns a Jetty server ready to be configured and the started. This server - * is only available when the test method has been annotated with - * {@link TestJetty}. Refer to {@link HTestCase} header for details. - *

- * Once configured, the Jetty server should be started. The server will be - * automatically stopped when the test method ends. - * - * @return a Jetty server ready to be configured and the started. - */ - public static Server getJettyServer() { - Server server = TEST_SERVLET_TL.get(); - if (server == null) { - throw new IllegalStateException("This test does not use @TestJetty"); - } - return server; - } - - /** - * Returns the base URL (SCHEMA://HOST:PORT) of the test Jetty server - * (see {@link #getJettyServer()}) once started. - * - * @return the base URL (SCHEMA://HOST:PORT) of the test Jetty server. - */ - public static URL getJettyURL() { - Server server = TEST_SERVLET_TL.get(); - if (server == null) { - throw new IllegalStateException("This test does not use @TestJetty"); - } - try { - return new URL("http://" + server.getConnectors()[0].getHost() + ":" + server.getConnectors()[0].getPort()); - } catch (MalformedURLException ex) { - throw new RuntimeException("It should never happen, " + ex.getMessage(), ex); - } - } - -} From 109735f3d1f7325fbada0fac5de3eb1db473e7a7 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 26 Jul 2012 18:21:39 +0000 Subject: [PATCH 23/39] HADOOP-8623. hadoop jar command should respect HADOOP_OPTS. Contributed by Steven Willis. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366126 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-common-project/hadoop-common/src/main/bin/hadoop | 9 +++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b5b671c7c15..e40b933d468 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -181,6 +181,9 @@ Trunk (unreleased changes) HADOOP-8593. Add missed @Override annotations in Metric/Metrics2 package. (Brandon Li via suresh) + HADOOP-8623. hadoop jar command should respect HADOOP_OPTS. + (Steven Willis via suresh) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index ae99d69d0f4..486465aa139 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -96,33 +96,30 @@ case $COMMAND in # the core commands if [ "$COMMAND" = "fs" ] ; then CLASS=org.apache.hadoop.fs.FsShell - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "version" ] ; then CLASS=org.apache.hadoop.util.VersionInfo - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "jar" ] ; then CLASS=org.apache.hadoop.util.RunJar elif [ "$COMMAND" = "distcp" ] ; then CLASS=org.apache.hadoop.tools.DistCp CLASSPATH=${CLASSPATH}:${TOOL_PATH} - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "daemonlog" ] ; then CLASS=org.apache.hadoop.log.LogLevel - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "archive" ] ; then CLASS=org.apache.hadoop.tools.HadoopArchives CLASSPATH=${CLASSPATH}:${TOOL_PATH} - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [[ "$COMMAND" = -* ]] ; then # class and package names cannot begin with a - echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'" exit 1 else - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" CLASS=$COMMAND fi shift + # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" + #make sure security appender is turned off HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}" From 2f478ac89cb592165ab462cd52235814507df9fe Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 26 Jul 2012 18:22:00 +0000 Subject: [PATCH 24/39] HADOOP-8642. ProtobufRpcEngine should log all RPCs if TRACE logging is enabled. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366127 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/ipc/Client.java | 5 ++++ .../apache/hadoop/ipc/ProtobufRpcEngine.java | 25 +++++++++++++++++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e40b933d468..83b52fdf281 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -88,6 +88,9 @@ Trunk (unreleased changes) HADOOP-8523. test-patch.sh doesn't validate patches before building (Jack Dintruff via jeagles) + HADOOP-8642. ProtobufRpcEngine should log all RPCs if TRACE logging is + enabled (todd) + BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index a923186511a..b0f5c93f757 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -1399,5 +1399,10 @@ public class Client { result = PRIME * result + ((ticket == null) ? 0 : ticket.hashCode()); return result; } + + @Override + public String toString() { + return serverPrincipal + "@" + address; + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java index 1bf6e4c55fb..96ca97817ab 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java @@ -51,13 +51,14 @@ import com.google.protobuf.BlockingService; import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.Message; import com.google.protobuf.ServiceException; +import com.google.protobuf.TextFormat; /** * RPC Engine for for protobuf based RPCs. */ @InterfaceStability.Evolving public class ProtobufRpcEngine implements RpcEngine { - private static final Log LOG = LogFactory.getLog(ProtobufRpcEngine.class); + public static final Log LOG = LogFactory.getLog(ProtobufRpcEngine.class); static { // Register the rpcRequest deserializer for WritableRpcEngine org.apache.hadoop.ipc.Server.registerProtocolEngine( @@ -191,16 +192,29 @@ public class ProtobufRpcEngine implements RpcEngine { HadoopRpcRequestProto rpcRequest = constructRpcRequest(method, args); RpcResponseWritable val = null; + + if (LOG.isTraceEnabled()) { + LOG.trace(Thread.currentThread().getId() + ": Call -> " + + remoteId + ": " + method.getName() + + " {" + TextFormat.shortDebugString((Message) args[1]) + "}"); + } try { val = (RpcResponseWritable) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER, new RpcRequestWritable(rpcRequest), remoteId); + } catch (Throwable e) { + if (LOG.isTraceEnabled()) { + LOG.trace(Thread.currentThread().getId() + ": Exception <- " + + remoteId + ": " + method.getName() + + " {" + e + "}"); + } + throw new ServiceException(e); } if (LOG.isDebugEnabled()) { long callTime = Time.now() - startTime; - LOG.debug("Call: " + method.getName() + " " + callTime); + LOG.debug("Call: " + method.getName() + " took " + callTime + "ms"); } Message prototype = null; @@ -213,6 +227,13 @@ public class ProtobufRpcEngine implements RpcEngine { try { returnMessage = prototype.newBuilderForType() .mergeFrom(val.responseMessage).build(); + + if (LOG.isTraceEnabled()) { + LOG.trace(Thread.currentThread().getId() + ": Response <- " + + remoteId + ": " + method.getName() + + " {" + TextFormat.shortDebugString(returnMessage) + "}"); + } + } catch (Throwable e) { throw new ServiceException(e); } From c1ea9b4490e7d6d030eeaeeff2fad3767d2cfd4a Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 27 Jul 2012 00:26:21 +0000 Subject: [PATCH 25/39] HDFS-3650. Use MutableQuantiles to provide latency histograms for various operations. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366246 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/conf/Configuration.java | 19 ++++ .../hadoop/metrics2/lib/MutableQuantiles.java | 6 +- .../apache/hadoop/net/SocketOutputStream.java | 18 +-- .../metrics2/lib/TestMutableMetrics.java | 4 +- .../apache/hadoop/test/MetricsAsserts.java | 25 +++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 + .../apache/hadoop/hdfs/DFSOutputStream.java | 11 +- .../hdfs/server/datanode/BlockSender.java | 10 +- .../datanode/metrics/DataNodeMetrics.java | 78 +++++++++++-- .../namenode/metrics/NameNodeMetrics.java | 37 ++++++- .../src/main/resources/hdfs-default.xml | 10 ++ .../server/datanode/TestDataNodeMetrics.java | 103 +++++++++++++----- .../namenode/metrics/TestNameNodeMetrics.java | 26 +++++ 14 files changed, 292 insertions(+), 59 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 37a6b73fe8d..83ac3867494 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -899,6 +899,25 @@ public class Configuration implements Iterable>, } return Integer.parseInt(valueString); } + + /** + * Get the value of the name property as a set of comma-delimited + * int values. + * + * If no such property exists, an empty array is returned. + * + * @param name property name + * @return property value interpreted as an array of comma-delimited + * int values + */ + public int[] getInts(String name) { + String[] strings = getTrimmedStrings(name); + int[] ints = new int[strings.length]; + for (int i = 0; i < strings.length; i++) { + ints[i] = Integer.parseInt(strings[i]); + } + return ints; + } /** * Set the value of the name property to an int. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java index 6830084c30b..e8992983a53 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java @@ -45,7 +45,8 @@ import com.google.common.annotations.VisibleForTesting; @InterfaceStability.Evolving public class MutableQuantiles extends MutableMetric { - static final Quantile[] quantiles = { new Quantile(0.50, 0.050), + @VisibleForTesting + public static final Quantile[] quantiles = { new Quantile(0.50, 0.050), new Quantile(0.75, 0.025), new Quantile(0.90, 0.010), new Quantile(0.95, 0.005), new Quantile(0.99, 0.001) }; @@ -90,8 +91,7 @@ public class MutableQuantiles extends MutableMetric { "Number of %s for %s with %ds interval", lsName, desc, interval)); // Construct the MetricsInfos for the quantiles, converting to percentiles quantileInfos = new MetricsInfo[quantiles.length]; - String nameTemplate = ucName + "%dthPercentile" + interval + "sInterval" - + uvName; + String nameTemplate = ucName + "%dthPercentile" + uvName; String descTemplate = "%d percentile " + lvName + " with " + interval + " second interval for " + desc; for (int i = 0; i < quantiles.length; i++) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java index 917d5f43f18..e7bfadeeda9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java @@ -31,8 +31,8 @@ import java.nio.channels.WritableByteChannel; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.metrics2.lib.MutableRate; -import org.apache.hadoop.util.Progressable; /** * This implements an output stream that can have a timeout while writing. @@ -179,9 +179,9 @@ public class SocketOutputStream extends OutputStream * @param fileCh FileChannel to transfer data from. * @param position position within the channel where the transfer begins * @param count number of bytes to transfer. - * @param waitForWritableTime updated by the nanoseconds spent waiting for - * the socket to become writable - * @param transferTime updated by the nanoseconds spent transferring data + * @param waitForWritableTime nanoseconds spent waiting for the socket + * to become writable + * @param transferTime nanoseconds spent transferring data * * @throws EOFException * If end of input file is reached before requested number of @@ -195,8 +195,8 @@ public class SocketOutputStream extends OutputStream * {@link FileChannel#transferTo(long, long, WritableByteChannel)}. */ public void transferToFully(FileChannel fileCh, long position, int count, - MutableRate waitForWritableTime, - MutableRate transferToTime) throws IOException { + LongWritable waitForWritableTime, + LongWritable transferToTime) throws IOException { long waitTime = 0; long transferTime = 0; while (count > 0) { @@ -236,12 +236,12 @@ public class SocketOutputStream extends OutputStream waitTime += wait - start; transferTime += transfer - wait; } - + if (waitForWritableTime != null) { - waitForWritableTime.add(waitTime); + waitForWritableTime.set(waitTime); } if (transferToTime != null) { - transferToTime.add(transferTime); + transferToTime.set(transferTime); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java index 023facffe89..ed83000390e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java @@ -150,7 +150,7 @@ public class TestMutableMetrics { info("FooNumOps", "Number of ops for stat with 5s interval"), (long) 2000); Quantile[] quants = MutableQuantiles.quantiles; - String name = "Foo%dthPercentile5sIntervalLatency"; + String name = "Foo%dthPercentileLatency"; String desc = "%d percentile latency with 5 second interval for stat"; for (Quantile q : quants) { int percentile = (int) (100 * q.quantile); @@ -176,7 +176,7 @@ public class TestMutableMetrics { "Latency", 5); Quantile[] quants = MutableQuantiles.quantiles; - String name = "Foo%dthPercentile5sIntervalLatency"; + String name = "Foo%dthPercentileLatency"; String desc = "%d percentile latency with 5 second interval for stat"; // Push values for three intervals diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java index f4bcb499e41..5a4b8a346cb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java @@ -23,7 +23,9 @@ import static com.google.common.base.Preconditions.*; import org.hamcrest.Description; import org.junit.Assert; +import static org.mockito.AdditionalMatchers.geq; import static org.mockito.Mockito.*; + import org.mockito.stubbing.Answer; import org.mockito.internal.matchers.GreaterThan; import org.mockito.invocation.InvocationOnMock; @@ -39,7 +41,11 @@ import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableQuantiles; +import org.apache.hadoop.metrics2.util.Quantile; + import static org.apache.hadoop.metrics2.lib.Interns.*; +import static org.apache.hadoop.test.MetricsAsserts.eqName; /** * Helpers for metrics source tests @@ -328,4 +334,23 @@ public class MetricsAsserts { MetricsSource source) { assertGaugeGt(name, greater, getMetrics(source)); } + + /** + * Asserts that the NumOps and quantiles for a metric have been changed at + * some point to a non-zero value. + * + * @param prefix of the metric + * @param rb MetricsRecordBuilder with the metric + */ + public static void assertQuantileGauges(String prefix, + MetricsRecordBuilder rb) { + verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0l)); + for (Quantile q : MutableQuantiles.quantiles) { + String nameTemplate = prefix + "%dthPercentileLatency"; + int percentile = (int) (100 * q.quantile); + verify(rb).addGauge( + eqName(info(String.format(nameTemplate, percentile), "")), + geq(0l)); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d90dda13578..92dccfa4dca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -356,6 +356,9 @@ Branch-2 ( Unreleased changes ) HDFS-3711. Manually convert remaining tests to JUnit4. (Andrew Wang via atm) + HDFS-3650. Use MutableQuantiles to provide latency histograms for various + operations. (Andrew Wang via atm) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index f97c9034e4d..43c6fd3a625 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -203,6 +203,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_CLIENT_READ_PREFETCH_SIZE_KEY = "dfs.client.read.prefetch.size"; public static final String DFS_CLIENT_RETRY_WINDOW_BASE= "dfs.client.retry.window.base"; public static final String DFS_METRICS_SESSION_ID_KEY = "dfs.metrics.session-id"; + public static final String DFS_METRICS_PERCENTILES_INTERVALS_KEY = "dfs.metrics.percentiles.intervals"; public static final String DFS_DATANODE_HOST_NAME_KEY = "dfs.datanode.hostname"; public static final String DFS_NAMENODE_HOSTS_KEY = "dfs.namenode.hosts"; public static final String DFS_NAMENODE_HOSTS_EXCLUDE_KEY = "dfs.namenode.hosts.exclude"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 949f3e7fdbf..f0dad469795 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -76,6 +76,8 @@ import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; +import com.google.common.annotations.VisibleForTesting; + /**************************************************************** * DFSOutputStream creates files from a stream of bytes. @@ -1210,7 +1212,8 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { // // returns the list of targets, if any, that is being currently used. // - synchronized DatanodeInfo[] getPipeline() { + @VisibleForTesting + public synchronized DatanodeInfo[] getPipeline() { if (streamer == null) { return null; } @@ -1752,11 +1755,13 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { } } - void setArtificialSlowdown(long period) { + @VisibleForTesting + public void setArtificialSlowdown(long period) { artificialSlowdown = period; } - synchronized void setChunksPerPacket(int value) { + @VisibleForTesting + public synchronized void setChunksPerPacket(int value) { chunksPerPacket = Math.min(chunksPerPacket, value); packetSize = PacketHeader.PKT_HEADER_LEN + (checksum.getBytesPerChecksum() + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index 80fe070be64..133a550bd9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; import org.apache.hadoop.io.nativeio.NativeIO; @@ -486,11 +487,14 @@ class BlockSender implements java.io.Closeable { // no need to flush since we know out is not a buffered stream FileChannel fileCh = ((FileInputStream)blockIn).getChannel(); + LongWritable waitTime = new LongWritable(); + LongWritable transferTime = new LongWritable(); sockOut.transferToFully(fileCh, blockInPosition, dataLen, - datanode.metrics.getSendDataPacketBlockedOnNetworkNanos(), - datanode.metrics.getSendDataPacketTransferNanos()); + waitTime, transferTime); + datanode.metrics.addSendDataPacketBlockedOnNetworkNanos(waitTime.get()); + datanode.metrics.addSendDataPacketTransferNanos(transferTime.get()); blockInPosition += dataLen; - } else { + } else { // normal transfer out.write(buf, 0, dataOff + dataLen); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java index 70b49d2bb1e..a9237c59106 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java @@ -29,6 +29,7 @@ import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.metrics2.source.JvmMetrics; @@ -74,19 +75,54 @@ public class DataNodeMetrics { @Metric MutableRate heartbeats; @Metric MutableRate blockReports; @Metric MutableRate packetAckRoundTripTimeNanos; - + MutableQuantiles[] packetAckRoundTripTimeNanosQuantiles; + @Metric MutableRate flushNanos; + MutableQuantiles[] flushNanosQuantiles; + @Metric MutableRate fsyncNanos; + MutableQuantiles[] fsyncNanosQuantiles; @Metric MutableRate sendDataPacketBlockedOnNetworkNanos; + MutableQuantiles[] sendDataPacketBlockedOnNetworkNanosQuantiles; @Metric MutableRate sendDataPacketTransferNanos; + MutableQuantiles[] sendDataPacketTransferNanosQuantiles; + final MetricsRegistry registry = new MetricsRegistry("datanode"); final String name; - public DataNodeMetrics(String name, String sessionId) { + public DataNodeMetrics(String name, String sessionId, int[] intervals) { this.name = name; registry.tag(SessionId, sessionId); + + final int len = intervals.length; + packetAckRoundTripTimeNanosQuantiles = new MutableQuantiles[len]; + flushNanosQuantiles = new MutableQuantiles[len]; + fsyncNanosQuantiles = new MutableQuantiles[len]; + sendDataPacketBlockedOnNetworkNanosQuantiles = new MutableQuantiles[len]; + sendDataPacketTransferNanosQuantiles = new MutableQuantiles[len]; + + for (int i = 0; i < len; i++) { + int interval = intervals[i]; + packetAckRoundTripTimeNanosQuantiles[i] = registry.newQuantiles( + "packetAckRoundTripTimeNanos" + interval + "s", + "Packet Ack RTT in ns", "ops", "latency", interval); + flushNanosQuantiles[i] = registry.newQuantiles( + "flushNanos" + interval + "s", + "Disk flush latency in ns", "ops", "latency", interval); + fsyncNanosQuantiles[i] = registry.newQuantiles( + "fsyncNanos" + interval + "s", "Disk fsync latency in ns", + "ops", "latency", interval); + sendDataPacketBlockedOnNetworkNanosQuantiles[i] = registry.newQuantiles( + "sendDataPacketBlockedOnNetworkNanos" + interval + "s", + "Time blocked on network while sending a packet in ns", + "ops", "latency", interval); + sendDataPacketTransferNanosQuantiles[i] = registry.newQuantiles( + "sendDataPacketTransferNanos" + interval + "s", + "Time reading from disk and writing to network while sending " + + "a packet in ns", "ops", "latency", interval); + } } public static DataNodeMetrics create(Configuration conf, String dnName) { @@ -94,8 +130,15 @@ public class DataNodeMetrics { MetricsSystem ms = DefaultMetricsSystem.instance(); JvmMetrics.create("DataNode", sessionId, ms); String name = "DataNodeActivity-"+ (dnName.isEmpty() - ? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt() : dnName.replace(':', '-')); - return ms.register(name, null, new DataNodeMetrics(name, sessionId)); + ? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt() + : dnName.replace(':', '-')); + + // Percentile measurement is off by default, by watching no intervals + int[] intervals = + conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY); + + return ms.register(name, null, new DataNodeMetrics(name, sessionId, + intervals)); } public String name() { return name; } @@ -166,14 +209,23 @@ public class DataNodeMetrics { public void addPacketAckRoundTripTimeNanos(long latencyNanos) { packetAckRoundTripTimeNanos.add(latencyNanos); + for (MutableQuantiles q : packetAckRoundTripTimeNanosQuantiles) { + q.add(latencyNanos); + } } public void addFlushNanos(long latencyNanos) { flushNanos.add(latencyNanos); + for (MutableQuantiles q : flushNanosQuantiles) { + q.add(latencyNanos); + } } public void addFsyncNanos(long latencyNanos) { fsyncNanos.add(latencyNanos); + for (MutableQuantiles q : fsyncNanosQuantiles) { + q.add(latencyNanos); + } } public void shutdown() { @@ -196,12 +248,18 @@ public class DataNodeMetrics { public void incrBlocksGetLocalPathInfo() { blocksGetLocalPathInfo.incr(); } - - public MutableRate getSendDataPacketBlockedOnNetworkNanos() { - return sendDataPacketBlockedOnNetworkNanos; + + public void addSendDataPacketBlockedOnNetworkNanos(long latencyNanos) { + sendDataPacketBlockedOnNetworkNanos.add(latencyNanos); + for (MutableQuantiles q : sendDataPacketBlockedOnNetworkNanosQuantiles) { + q.add(latencyNanos); + } } - - public MutableRate getSendDataPacketTransferNanos() { - return sendDataPacketTransferNanos; + + public void addSendDataPacketTransferNanos(long latencyNanos) { + sendDataPacketTransferNanos.add(latencyNanos); + for (MutableQuantiles q : sendDataPacketTransferNanosQuantiles) { + q.add(latencyNanos); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index a75701ef86b..ecfe98697ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -17,17 +17,20 @@ */ package org.apache.hadoop.hdfs.server.namenode.metrics; +import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; +import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; + import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; -import static org.apache.hadoop.metrics2.impl.MsInfo.*; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.metrics2.source.JvmMetrics; @@ -57,15 +60,31 @@ public class NameNodeMetrics { @Metric("Journal transactions") MutableRate transactions; @Metric("Journal syncs") MutableRate syncs; + MutableQuantiles[] syncsQuantiles; @Metric("Journal transactions batched in sync") MutableCounterLong transactionsBatchedInSync; @Metric("Block report") MutableRate blockReport; + MutableQuantiles[] blockReportQuantiles; @Metric("Duration in SafeMode at startup") MutableGaugeInt safeModeTime; @Metric("Time loading FS Image at startup") MutableGaugeInt fsImageLoadTime; - NameNodeMetrics(String processName, String sessionId) { + NameNodeMetrics(String processName, String sessionId, int[] intervals) { registry.tag(ProcessName, processName).tag(SessionId, sessionId); + + final int len = intervals.length; + syncsQuantiles = new MutableQuantiles[len]; + blockReportQuantiles = new MutableQuantiles[len]; + + for (int i = 0; i < len; i++) { + int interval = intervals[i]; + syncsQuantiles[i] = registry.newQuantiles( + "syncs" + interval + "s", + "Journal syncs", "ops", "latency", interval); + blockReportQuantiles[i] = registry.newQuantiles( + "blockReport" + interval + "s", + "Block report", "ops", "latency", interval); + } } public static NameNodeMetrics create(Configuration conf, NamenodeRole r) { @@ -73,7 +92,11 @@ public class NameNodeMetrics { String processName = r.toString(); MetricsSystem ms = DefaultMetricsSystem.instance(); JvmMetrics.create(processName, sessionId, ms); - return ms.register(new NameNodeMetrics(processName, sessionId)); + + // Percentile measurement is off by default, by watching no intervals + int[] intervals = + conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY); + return ms.register(new NameNodeMetrics(processName, sessionId, intervals)); } public void shutdown() { @@ -146,6 +169,9 @@ public class NameNodeMetrics { public void addSync(long elapsed) { syncs.add(elapsed); + for (MutableQuantiles q : syncsQuantiles) { + q.add(elapsed); + } } public void setFsImageLoadTime(long elapsed) { @@ -154,6 +180,9 @@ public class NameNodeMetrics { public void addBlockReport(long latency) { blockReport.add(latency); + for (MutableQuantiles q : blockReportQuantiles) { + q.add(latency); + } } public void setSafeModeTime(long elapsed) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 4815ea147ff..fe6cbd57bb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1004,4 +1004,14 @@ + + dfs.metrics.percentiles.intervals + + + Comma-delimited set of integers denoting the desired rollover intervals + (in seconds) for percentile latency metrics on the Namenode and Datanode. + By default, percentile latency metrics are disabled. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java index 81748ba2cff..4921f0307eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java @@ -18,21 +18,26 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.util.List; -import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.junit.Test; @@ -59,8 +64,10 @@ public class TestDataNodeMetrics { } @Test - public void testSendDataPacket() throws Exception { + public void testSendDataPacketMetrics() throws Exception { Configuration conf = new HdfsConfiguration(); + final int interval = 1; + conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { FileSystem fs = cluster.getFileSystem(); @@ -73,64 +80,110 @@ public class TestDataNodeMetrics { assertEquals(datanodes.size(), 1); DataNode datanode = datanodes.get(0); MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name()); - // Expect 2 packets, 1 for the 1 byte read, 1 for the empty packet // signaling the end of the block assertCounter("SendDataPacketTransferNanosNumOps", (long)2, rb); assertCounter("SendDataPacketBlockedOnNetworkNanosNumOps", (long)2, rb); + // Wait for at least 1 rollover + Thread.sleep((interval + 1) * 1000); + // Check that the sendPacket percentiles rolled to non-zero values + String sec = interval + "s"; + assertQuantileGauges("SendDataPacketBlockedOnNetworkNanos" + sec, rb); + assertQuantileGauges("SendDataPacketTransferNanos" + sec, rb); } finally { if (cluster != null) {cluster.shutdown();} } } @Test - public void testFlushMetric() throws Exception { + public void testReceivePacketMetrics() throws Exception { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + final int interval = 1; + conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { cluster.waitActive(); DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); Path testFile = new Path("/testFlushNanosMetric.txt"); - DFSTestUtil.createFile(fs, testFile, 1, (short)1, new Random().nextLong()); - + FSDataOutputStream fout = fs.create(testFile); + fout.write(new byte[1]); + fout.hsync(); + fout.close(); List datanodes = cluster.getDataNodes(); DataNode datanode = datanodes.get(0); MetricsRecordBuilder dnMetrics = getMetrics(datanode.getMetrics().name()); - // Expect 2 flushes, 1 for the flush that occurs after writing, 1 that occurs - // on closing the data and metadata files. + // Expect two flushes, 1 for the flush that occurs after writing, + // 1 that occurs on closing the data and metadata files. assertCounter("FlushNanosNumOps", 2L, dnMetrics); + // Expect two syncs, one from the hsync, one on close. + assertCounter("FsyncNanosNumOps", 2L, dnMetrics); + // Wait for at least 1 rollover + Thread.sleep((interval + 1) * 1000); + // Check the receivePacket percentiles that should be non-zero + String sec = interval + "s"; + assertQuantileGauges("FlushNanos" + sec, dnMetrics); + assertQuantileGauges("FsyncNanos" + sec, dnMetrics); } finally { if (cluster != null) {cluster.shutdown();} } } + /** + * Tests that round-trip acks in a datanode write pipeline are correctly + * measured. + */ @Test public void testRoundTripAckMetric() throws Exception { - final int DATANODE_COUNT = 2; - + final int datanodeCount = 2; + final int interval = 1; Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build(); + conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes( + datanodeCount).build(); try { cluster.waitActive(); - DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); - + FileSystem fs = cluster.getFileSystem(); + // Open a file and get the head of the pipeline Path testFile = new Path("/testRoundTripAckMetric.txt"); - DFSTestUtil.createFile(fs, testFile, 1, (short)DATANODE_COUNT, - new Random().nextLong()); - - boolean foundNonzeroPacketAckNumOps = false; + FSDataOutputStream fsout = fs.create(testFile, (short) datanodeCount); + DFSOutputStream dout = (DFSOutputStream) fsout.getWrappedStream(); + // Slow down the writes to catch the write pipeline + dout.setChunksPerPacket(5); + dout.setArtificialSlowdown(3000); + fsout.write(new byte[10000]); + DatanodeInfo[] pipeline = null; + int count = 0; + while (pipeline == null && count < 5) { + pipeline = dout.getPipeline(); + System.out.println("Waiting for pipeline to be created."); + Thread.sleep(1000); + count++; + } + // Get the head node that should be receiving downstream acks + DatanodeInfo headInfo = pipeline[0]; + DataNode headNode = null; for (DataNode datanode : cluster.getDataNodes()) { - MetricsRecordBuilder dnMetrics = getMetrics(datanode.getMetrics().name()); - if (getLongCounter("PacketAckRoundTripTimeNanosNumOps", dnMetrics) > 0) { - foundNonzeroPacketAckNumOps = true; + if (datanode.getDatanodeId().equals(headInfo)) { + headNode = datanode; + break; } } - assertTrue( - "Expected at least one datanode to have reported PacketAckRoundTripTimeNanos metric", - foundNonzeroPacketAckNumOps); + assertNotNull("Could not find the head of the datanode write pipeline", + headNode); + // Close the file and wait for the metrics to rollover + Thread.sleep((interval + 1) * 1000); + // Check the ack was received + MetricsRecordBuilder dnMetrics = getMetrics(headNode.getMetrics() + .name()); + assertTrue("Expected non-zero number of acks", + getLongCounter("PacketAckRoundTripTimeNanosNumOps", dnMetrics) > 0); + assertQuantileGauges("PacketAckRoundTripTimeNanos" + interval + + "s", dnMetrics); } finally { - if (cluster != null) {cluster.shutdown();} + if (cluster != null) { + cluster.shutdown(); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index d29bd4ecb5d..c4d659904e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode.metrics; import static org.apache.hadoop.test.MetricsAsserts.assertCounter; import static org.apache.hadoop.test.MetricsAsserts.assertGauge; +import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertTrue; @@ -63,6 +64,9 @@ public class TestNameNodeMetrics { // Number of datanodes in the cluster private static final int DATANODE_COUNT = 3; private static final int WAIT_GAUGE_VALUE_RETRIES = 20; + + // Rollover interval of percentile metrics (in seconds) + private static final int PERCENTILES_INTERVAL = 1; static { CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100); @@ -71,6 +75,8 @@ public class TestNameNodeMetrics { DFS_REPLICATION_INTERVAL); CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, DFS_REPLICATION_INTERVAL); + CONF.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, + "" + PERCENTILES_INTERVAL); ((Log4JLogger)LogFactory.getLog(MetricsAsserts.class)) .getLogger().setLevel(Level.DEBUG); @@ -352,4 +358,24 @@ public class TestNameNodeMetrics { assertGauge("TransactionsSinceLastCheckpoint", 1L, getMetrics(NS_METRICS)); assertGauge("TransactionsSinceLastLogRoll", 1L, getMetrics(NS_METRICS)); } + + /** + * Tests that the sync and block report metrics get updated on cluster + * startup. + */ + @Test + public void testSyncAndBlockReportMetric() throws Exception { + MetricsRecordBuilder rb = getMetrics(NN_METRICS); + // We have one sync when the cluster starts up, just opening the journal + assertCounter("SyncsNumOps", 1L, rb); + // Each datanode reports in when the cluster comes up + assertCounter("BlockReportNumOps", (long)DATANODE_COUNT, rb); + + // Sleep for an interval+slop to let the percentiles rollover + Thread.sleep((PERCENTILES_INTERVAL+1)*1000); + + // Check that the percentiles were updated + assertQuantileGauges("Syncs1s", rb); + assertQuantileGauges("BlockReport1s", rb); + } } From d87b545165f9442f614665521ce04424af1490e8 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 27 Jul 2012 01:48:09 +0000 Subject: [PATCH 26/39] MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366258 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop/mapreduce/task/reduce/Fetcher.java | 76 ++++--- .../mapreduce/task/reduce/TestFetcher.java | 188 ++++++++++++++++++ 3 files changed, 239 insertions(+), 28 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 2cc69e6c78d..620ccb84a33 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -757,6 +757,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4467. IndexCache failures due to missing synchronization (Kihwal Lee via tgraves) + MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans + via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java index 64bc43e51b6..6c527ae1ce5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java @@ -21,11 +21,12 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.net.HttpURLConnection; import java.net.URLConnection; import java.security.GeneralSecurityException; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -53,7 +54,8 @@ import org.apache.hadoop.mapreduce.task.reduce.MapOutput.Type; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; -@SuppressWarnings({"deprecation"}) +import com.google.common.annotations.VisibleForTesting; + class Fetcher extends Thread { private static final Log LOG = LogFactory.getLog(Fetcher.class); @@ -199,6 +201,7 @@ class Fetcher extends Thread { } } + @VisibleForTesting protected HttpURLConnection openConnection(URL url) throws IOException { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); if (sslShuffle) { @@ -209,17 +212,18 @@ class Fetcher extends Thread { throw new IOException(ex); } httpsConn.setHostnameVerifier(sslFactory.getHostnameVerifier()); - } - return conn; } - + return conn; + } + /** * The crux of the matter... * * @param host {@link MapHost} from which we need to * shuffle available map-outputs. */ - private void copyFromHost(MapHost host) throws IOException { + @VisibleForTesting + protected void copyFromHost(MapHost host) throws IOException { // Get completed maps on 'host' List maps = scheduler.getMapsForHost(host); @@ -229,9 +233,9 @@ class Fetcher extends Thread { return; } - LOG.debug("Fetcher " + id + " going to fetch from " + host); - for (TaskAttemptID tmp: maps) { - LOG.debug(tmp); + if(LOG.isDebugEnabled()) { + LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: " + + maps); } // List of maps to be fetched yet @@ -304,17 +308,25 @@ class Fetcher extends Thread { try { // Loop through available map-outputs and fetch them - // On any error, good becomes false and we exit after putting back - // the remaining maps to the yet_to_be_fetched list - boolean good = true; - while (!remaining.isEmpty() && good) { - good = copyMapOutput(host, input, remaining); + // On any error, faildTasks is not null and we exit + // after putting back the remaining maps to the + // yet_to_be_fetched list and marking the failed tasks. + TaskAttemptID[] failedTasks = null; + while (!remaining.isEmpty() && failedTasks == null) { + failedTasks = copyMapOutput(host, input, remaining); + } + + if(failedTasks != null && failedTasks.length > 0) { + LOG.warn("copyMapOutput failed for tasks "+Arrays.toString(failedTasks)); + for(TaskAttemptID left: failedTasks) { + scheduler.copyFailed(left, host, true); + } } IOUtils.cleanup(LOG, input); // Sanity check - if (good && !remaining.isEmpty()) { + if (failedTasks == null && !remaining.isEmpty()) { throw new IOException("server didn't return all expected map outputs: " + remaining.size() + " left."); } @@ -323,10 +335,11 @@ class Fetcher extends Thread { scheduler.putBackKnownMapOutput(host, left); } } - - } + } - private boolean copyMapOutput(MapHost host, + private static TaskAttemptID[] EMPTY_ATTEMPT_ID_ARRAY = new TaskAttemptID[0]; + + private TaskAttemptID[] copyMapOutput(MapHost host, DataInputStream input, Set remaining) { MapOutput mapOutput = null; @@ -348,18 +361,21 @@ class Fetcher extends Thread { } catch (IllegalArgumentException e) { badIdErrs.increment(1); LOG.warn("Invalid map id ", e); - return false; + //Don't know which one was bad, so consider all of them as bad + return remaining.toArray(new TaskAttemptID[remaining.size()]); } // Do some basic sanity verification if (!verifySanity(compressedLength, decompressedLength, forReduce, remaining, mapId)) { - return false; + return new TaskAttemptID[] {mapId}; } - LOG.debug("header: " + mapId + ", len: " + compressedLength + - ", decomp len: " + decompressedLength); + if(LOG.isDebugEnabled()) { + LOG.debug("header: " + mapId + ", len: " + compressedLength + + ", decomp len: " + decompressedLength); + } // Get the location for the map output - either in-memory or on-disk mapOutput = merger.reserve(mapId, decompressedLength, id); @@ -367,7 +383,8 @@ class Fetcher extends Thread { // Check if we can shuffle *now* ... if (mapOutput.getType() == Type.WAIT) { LOG.info("fetcher#" + id + " - MergerManager returned Status.WAIT ..."); - return false; + //Not an error but wait to process data. + return EMPTY_ATTEMPT_ID_ARRAY; } // Go! @@ -389,24 +406,27 @@ class Fetcher extends Thread { // Note successful shuffle remaining.remove(mapId); metrics.successFetch(); - return true; + return null; } catch (IOException ioe) { ioErrs.increment(1); if (mapId == null || mapOutput == null) { LOG.info("fetcher#" + id + " failed to read map header" + mapId + " decomp: " + decompressedLength + ", " + compressedLength, ioe); - return false; + if(mapId == null) { + return remaining.toArray(new TaskAttemptID[remaining.size()]); + } else { + return new TaskAttemptID[] {mapId}; + } } - LOG.info("Failed to shuffle output of " + mapId + + LOG.warn("Failed to shuffle output of " + mapId + " from " + host.getHostName(), ioe); // Inform the shuffle-scheduler mapOutput.abort(); - scheduler.copyFailed(mapId, host, true); metrics.failedFetch(); - return false; + return new TaskAttemptID[] {mapId}; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java new file mode 100644 index 00000000000..097f120dd21 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.task.reduce; + +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; + +import javax.crypto.SecretKey; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.Counters; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; +import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; +import org.junit.Test; + +/** + * Test that the Fetcher does what we expect it to. + */ +public class TestFetcher { + private static final Log LOG = LogFactory.getLog(TestFetcher.class); + + public static class FakeFetcher extends Fetcher { + + private HttpURLConnection connection; + + public FakeFetcher(JobConf job, TaskAttemptID reduceId, + ShuffleScheduler scheduler, MergeManager merger, Reporter reporter, + ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, + SecretKey jobTokenSecret, HttpURLConnection connection) { + super(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, + jobTokenSecret); + this.connection = connection; + } + + @Override + protected HttpURLConnection openConnection(URL url) throws IOException { + if(connection != null) { + return connection; + } + return super.openConnection(url); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testCopyFromHostBogusHeader() throws Exception { + LOG.info("testCopyFromHostBogusHeader"); + JobConf job = new JobConf(); + TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1"); + ShuffleScheduler ss = mock(ShuffleScheduler.class); + MergeManager mm = mock(MergeManager.class); + Reporter r = mock(Reporter.class); + ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class); + ExceptionReporter except = mock(ExceptionReporter.class); + SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0}); + HttpURLConnection connection = mock(HttpURLConnection.class); + + Counters.Counter allErrs = mock(Counters.Counter.class); + when(r.getCounter(anyString(), anyString())) + .thenReturn(allErrs); + + Fetcher underTest = new FakeFetcher(job, id, ss, mm, + r, metrics, except, key, connection); + + + MapHost host = new MapHost("localhost", "http://localhost:8080/"); + + ArrayList maps = new ArrayList(1); + TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1"); + maps.add(map1ID); + TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1"); + maps.add(map2ID); + when(ss.getMapsForHost(host)).thenReturn(maps); + + String encHash = "vFE234EIFCiBgYs2tCXY/SjT8Kg="; + String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); + + when(connection.getResponseCode()).thenReturn(200); + when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)) + .thenReturn(replyHash); + ByteArrayInputStream in = new ByteArrayInputStream( + "\u00010 BOGUS DATA\nBOGUS DATA\nBOGUS DATA\n".getBytes()); + when(connection.getInputStream()).thenReturn(in); + + underTest.copyFromHost(host); + + verify(connection) + .addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, + encHash); + + verify(allErrs).increment(1); + verify(ss).copyFailed(map1ID, host, true); + verify(ss).copyFailed(map2ID, host, true); + + verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); + verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); + } + + @SuppressWarnings("unchecked") + @Test + public void testCopyFromHostWait() throws Exception { + LOG.info("testCopyFromHostWait"); + JobConf job = new JobConf(); + TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1"); + ShuffleScheduler ss = mock(ShuffleScheduler.class); + MergeManager mm = mock(MergeManager.class); + Reporter r = mock(Reporter.class); + ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class); + ExceptionReporter except = mock(ExceptionReporter.class); + SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0}); + HttpURLConnection connection = mock(HttpURLConnection.class); + + Counters.Counter allErrs = mock(Counters.Counter.class); + when(r.getCounter(anyString(), anyString())) + .thenReturn(allErrs); + + Fetcher underTest = new FakeFetcher(job, id, ss, mm, + r, metrics, except, key, connection); + + + MapHost host = new MapHost("localhost", "http://localhost:8080/"); + + ArrayList maps = new ArrayList(1); + TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1"); + maps.add(map1ID); + TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1"); + maps.add(map2ID); + when(ss.getMapsForHost(host)).thenReturn(maps); + + String encHash = "vFE234EIFCiBgYs2tCXY/SjT8Kg="; + String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); + + when(connection.getResponseCode()).thenReturn(200); + when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)) + .thenReturn(replyHash); + ShuffleHeader header = new ShuffleHeader(map1ID.toString(), 10, 10, 1); + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + header.write(new DataOutputStream(bout)); + ByteArrayInputStream in = new ByteArrayInputStream(bout.toByteArray()); + when(connection.getInputStream()).thenReturn(in); + //Defaults to WAIT, which is what we want to test + MapOutput mapOut = new MapOutput(map1ID); + when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt())) + .thenReturn(mapOut); + + underTest.copyFromHost(host); + + verify(connection) + .addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, + encHash); + verify(allErrs, never()).increment(1); + verify(ss, never()).copyFailed(map1ID, host, true); + verify(ss, never()).copyFailed(map2ID, host, true); + + verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); + verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); + } + +} From 9c5bd764fc3faf995c4d917bec86806eda2747f7 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 27 Jul 2012 15:13:57 +0000 Subject: [PATCH 27/39] HADOOP-8626. Typo in default setting for hadoop.security.group.mapping.ldap.search.filter.user. Contributed by Jonathan Natkins. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366410 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++++ .../hadoop-common/src/main/resources/core-default.xml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 83b52fdf281..03d221c50d0 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -367,6 +367,10 @@ Branch-2 ( Unreleased changes ) HADOOP-8537. Fix TFile tests to pass even when native zlib support is not compiled. (todd) + HADOOP-8626. Typo in default setting for + hadoop.security.group.mapping.ldap.search.filter.user. (Jonathan Natkins + via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 7667eecb244..93b11c008b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -165,7 +165,7 @@ hadoop.security.group.mapping.ldap.search.filter.user - (&(objectClass=user)(sAMAccountName={0}) + (&(objectClass=user)(sAMAccountName={0})) An additional filter to use when searching for LDAP users. The default will usually be appropriate for Active Directory installations. If connecting to From b3b72482e4476fe6ead484e85b55c766bcf998d0 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Fri, 27 Jul 2012 16:39:51 +0000 Subject: [PATCH 28/39] HADOOP-8613. AbstractDelegationTokenIdentifier#getUser() should set token auth type. (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366440 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../AbstractDelegationTokenIdentifier.java | 12 +++-- .../token/delegation/TestDelegationToken.java | 48 +++++++++++++++++++ .../hadoop/hdfs/server/common/JspHelper.java | 1 - 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 03d221c50d0..f01b8a5db2e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -854,6 +854,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8551. fs -mkdir creates parent directories without the -p option (John George via bobby) + HADOOP-8613. AbstractDelegationTokenIdentifier#getUser() should set token + auth type. (daryn) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java index 8c3c1b2d35f..b3e367bdf25 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.security.HadoopKerberosName; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.TokenIdentifier; import com.google.common.annotations.VisibleForTesting; @@ -88,14 +89,17 @@ extends TokenIdentifier { if ( (owner == null) || ("".equals(owner.toString()))) { return null; } + final UserGroupInformation realUgi; + final UserGroupInformation ugi; if ((realUser == null) || ("".equals(realUser.toString())) || realUser.equals(owner)) { - return UserGroupInformation.createRemoteUser(owner.toString()); + ugi = realUgi = UserGroupInformation.createRemoteUser(owner.toString()); } else { - UserGroupInformation realUgi = UserGroupInformation - .createRemoteUser(realUser.toString()); - return UserGroupInformation.createProxyUser(owner.toString(), realUgi); + realUgi = UserGroupInformation.createRemoteUser(realUser.toString()); + ugi = UserGroupInformation.createProxyUser(owner.toString(), realUgi); } + realUgi.setAuthenticationMethod(AuthenticationMethod.TOKEN); + return ugi; } public Text getOwner() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java index ed07c972b56..c1dd00a4d7d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java @@ -40,6 +40,8 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -171,6 +173,52 @@ public class TestDelegationToken { } } + @Test + public void testGetUserNullOwner() { + TestDelegationTokenIdentifier ident = + new TestDelegationTokenIdentifier(null, null, null); + UserGroupInformation ugi = ident.getUser(); + assertNull(ugi); + } + + @Test + public void testGetUserWithOwner() { + TestDelegationTokenIdentifier ident = + new TestDelegationTokenIdentifier(new Text("owner"), null, null); + UserGroupInformation ugi = ident.getUser(); + assertNull(ugi.getRealUser()); + assertEquals("owner", ugi.getUserName()); + assertEquals(AuthenticationMethod.TOKEN, ugi.getAuthenticationMethod()); + } + + @Test + public void testGetUserWithOwnerEqualsReal() { + Text owner = new Text("owner"); + TestDelegationTokenIdentifier ident = + new TestDelegationTokenIdentifier(owner, null, owner); + UserGroupInformation ugi = ident.getUser(); + assertNull(ugi.getRealUser()); + assertEquals("owner", ugi.getUserName()); + assertEquals(AuthenticationMethod.TOKEN, ugi.getAuthenticationMethod()); + } + + @Test + public void testGetUserWithOwnerAndReal() { + Text owner = new Text("owner"); + Text realUser = new Text("realUser"); + TestDelegationTokenIdentifier ident = + new TestDelegationTokenIdentifier(owner, null, realUser); + UserGroupInformation ugi = ident.getUser(); + assertNotNull(ugi.getRealUser()); + assertNull(ugi.getRealUser().getRealUser()); + assertEquals("owner", ugi.getUserName()); + assertEquals("realUser", ugi.getRealUser().getUserName()); + assertEquals(AuthenticationMethod.PROXY, + ugi.getAuthenticationMethod()); + assertEquals(AuthenticationMethod.TOKEN, + ugi.getRealUser().getAuthenticationMethod()); + } + @Test public void testDelegationTokenSecretManager() throws Exception { final TestDelegationTokenSecretManager dtSecretManager = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java index f8bfee73a7e..d153a8f48c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java @@ -578,7 +578,6 @@ public class JspHelper { ProxyUsers.authorize(ugi, request.getRemoteAddr(), conf); } ugi.addToken(token); - ugi.setAuthenticationMethod(AuthenticationMethod.TOKEN); } else { if(remoteUser == null) { throw new IOException("Security enabled but user not " + From cbd59c1c50242a9ee799cfe9a33f3bdc4561c4ea Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Fri, 27 Jul 2012 18:02:56 +0000 Subject: [PATCH 29/39] HDFS-3553. Hftp proxy tokens are broken (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366471 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/server/common/JspHelper.java | 133 +++++---- .../hdfs/server/common/TestJspHelper.java | 263 ++++++++++++++++++ 3 files changed, 331 insertions(+), 67 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 92dccfa4dca..4c7d679ad67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1411,6 +1411,8 @@ Release 0.23.3 - UNRELEASED HDFS-3696. Set chunked streaming mode in WebHdfsFileSystem write operations to get around a Java library bug causing OutOfMemoryError. (szetszwo) + HDFS-3553. Hftp proxy tokens are broken (daryn) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java index d153a8f48c9..cce5f74d83c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java @@ -487,12 +487,17 @@ public class JspHelper { */ public static UserGroupInformation getDefaultWebUser(Configuration conf ) throws IOException { + return UserGroupInformation.createRemoteUser(getDefaultWebUserName(conf)); + } + + private static String getDefaultWebUserName(Configuration conf + ) throws IOException { String user = conf.get( HADOOP_HTTP_STATIC_USER, DEFAULT_HADOOP_HTTP_STATIC_USER); if (user == null || user.length() == 0) { throw new IOException("Cannot determine UGI from request or conf"); } - return UserGroupInformation.createRemoteUser(user); + return user; } private static InetSocketAddress getNNServiceAddress(ServletContext context, @@ -538,64 +543,45 @@ public class JspHelper { HttpServletRequest request, Configuration conf, final AuthenticationMethod secureAuthMethod, final boolean tryUgiParameter) throws IOException { - final UserGroupInformation ugi; + UserGroupInformation ugi = null; final String usernameFromQuery = getUsernameFromQuery(request, tryUgiParameter); final String doAsUserFromQuery = request.getParameter(DoAsParam.NAME); - - if(UserGroupInformation.isSecurityEnabled()) { - final String remoteUser = request.getRemoteUser(); - String tokenString = request.getParameter(DELEGATION_PARAMETER_NAME); + final String remoteUser; + + if (UserGroupInformation.isSecurityEnabled()) { + remoteUser = request.getRemoteUser(); + final String tokenString = request.getParameter(DELEGATION_PARAMETER_NAME); if (tokenString != null) { - Token token = - new Token(); - token.decodeFromUrlString(tokenString); - InetSocketAddress serviceAddress = getNNServiceAddress(context, request); - if (serviceAddress != null) { - SecurityUtil.setTokenService(token, serviceAddress); - token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND); - } - ByteArrayInputStream buf = new ByteArrayInputStream(token - .getIdentifier()); - DataInputStream in = new DataInputStream(buf); - DelegationTokenIdentifier id = new DelegationTokenIdentifier(); - id.readFields(in); - if (context != null) { - final NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context); - if (nn != null) { - // Verify the token. - nn.getNamesystem().verifyToken(id, token.getPassword()); - } - } - ugi = id.getUser(); - if (ugi.getRealUser() == null) { - //non-proxy case - checkUsername(ugi.getShortUserName(), usernameFromQuery); - checkUsername(null, doAsUserFromQuery); - } else { - //proxy case - checkUsername(ugi.getRealUser().getShortUserName(), usernameFromQuery); - checkUsername(ugi.getShortUserName(), doAsUserFromQuery); - ProxyUsers.authorize(ugi, request.getRemoteAddr(), conf); - } - ugi.addToken(token); - } else { - if(remoteUser == null) { - throw new IOException("Security enabled but user not " + - "authenticated by filter"); - } - final UserGroupInformation realUgi = UserGroupInformation.createRemoteUser(remoteUser); - checkUsername(realUgi.getShortUserName(), usernameFromQuery); + // Token-based connections need only verify the effective user, and + // disallow proxying to different user. Proxy authorization checks + // are not required since the checks apply to issuing a token. + ugi = getTokenUGI(context, request, tokenString, conf); + checkUsername(ugi.getShortUserName(), usernameFromQuery); + checkUsername(ugi.getShortUserName(), doAsUserFromQuery); + } else if (remoteUser == null) { + throw new IOException( + "Security enabled but user not authenticated by filter"); + } + } else { + // Security's not on, pull from url or use default web user + remoteUser = (usernameFromQuery == null) + ? getDefaultWebUserName(conf) // not specified in request + : usernameFromQuery; + } + + if (ugi == null) { // security is off, or there's no token + ugi = UserGroupInformation.createRemoteUser(remoteUser); + checkUsername(ugi.getShortUserName(), usernameFromQuery); + if (UserGroupInformation.isSecurityEnabled()) { // This is not necessarily true, could have been auth'ed by user-facing // filter - realUgi.setAuthenticationMethod(secureAuthMethod); - ugi = initUGI(realUgi, doAsUserFromQuery, request, true, conf); + ugi.setAuthenticationMethod(secureAuthMethod); + } + if (doAsUserFromQuery != null) { + // create and attempt to authorize a proxy user + ugi = UserGroupInformation.createProxyUser(doAsUserFromQuery, ugi); + ProxyUsers.authorize(ugi, request.getRemoteAddr(), conf); } - } else { // Security's not on, pull from url - final UserGroupInformation realUgi = usernameFromQuery == null? - getDefaultWebUser(conf) // not specified in request - : UserGroupInformation.createRemoteUser(usernameFromQuery); - realUgi.setAuthenticationMethod(AuthenticationMethod.SIMPLE); - ugi = initUGI(realUgi, doAsUserFromQuery, request, false, conf); } if(LOG.isDebugEnabled()) @@ -603,21 +589,34 @@ public class JspHelper { return ugi; } - private static UserGroupInformation initUGI(final UserGroupInformation realUgi, - final String doAsUserFromQuery, final HttpServletRequest request, - final boolean isSecurityEnabled, final Configuration conf - ) throws AuthorizationException { - final UserGroupInformation ugi; - if (doAsUserFromQuery == null) { - //non-proxy case - ugi = realUgi; - } else { - //proxy case - ugi = UserGroupInformation.createProxyUser(doAsUserFromQuery, realUgi); - ugi.setAuthenticationMethod( - isSecurityEnabled? AuthenticationMethod.PROXY: AuthenticationMethod.SIMPLE); - ProxyUsers.authorize(ugi, request.getRemoteAddr(), conf); + private static UserGroupInformation getTokenUGI(ServletContext context, + HttpServletRequest request, + String tokenString, + Configuration conf) + throws IOException { + final Token token = + new Token(); + token.decodeFromUrlString(tokenString); + InetSocketAddress serviceAddress = getNNServiceAddress(context, request); + if (serviceAddress != null) { + SecurityUtil.setTokenService(token, serviceAddress); + token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND); } + + ByteArrayInputStream buf = + new ByteArrayInputStream(token.getIdentifier()); + DataInputStream in = new DataInputStream(buf); + DelegationTokenIdentifier id = new DelegationTokenIdentifier(); + id.readFields(in); + if (context != null) { + final NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context); + if (nn != null) { + // Verify the token. + nn.getNamesystem().verifyToken(id, token.getPassword()); + } + } + UserGroupInformation ugi = id.getUser(); + ugi.addToken(token); return ugi; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java index c7fafdb13bc..d024bcdbf7c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java @@ -31,8 +31,13 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; +import org.apache.hadoop.hdfs.web.resources.DoAsParam; +import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; @@ -136,4 +141,262 @@ public class TestJspHelper { Assert.assertEquals("", delegationTokenParam); } + @Test + public void testGetUgiFromToken() throws IOException { + conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/"); + ServletContext context = mock(ServletContext.class); + String realUser = "TheDoctor"; + String user = "TheNurse"; + conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); + UserGroupInformation ugi; + HttpServletRequest request; + + Text ownerText = new Text(user); + DelegationTokenIdentifier dtId = new DelegationTokenIdentifier( + ownerText, ownerText, new Text(realUser)); + Token token = new Token( + dtId, new DummySecretManager(0, 0, 0, 0)); + String tokenString = token.encodeToUrlString(); + + // token with no auth-ed user + request = getMockRequest(null, null, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // token with auth-ed user + request = getMockRequest(realUser, null, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // completely different user, token trumps auth + request = getMockRequest("rogue", null, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // expected case + request = getMockRequest(null, user, null); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromToken(ugi); + + // can't proxy with a token! + request = getMockRequest(null, null, "rogue"); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name=rogue != expected="+user, + ioe.getMessage()); + } + + // can't proxy with a token! + request = getMockRequest(null, user, "rogue"); + when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn( + tokenString); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name=rogue != expected="+user, + ioe.getMessage()); + } + } + + @Test + public void testGetNonProxyUgi() throws IOException { + conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/"); + ServletContext context = mock(ServletContext.class); + String realUser = "TheDoctor"; + String user = "TheNurse"; + conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); + UserGroupInformation ugi; + HttpServletRequest request; + + // have to be auth-ed with remote user + request = getMockRequest(null, null, null); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + request = getMockRequest(null, realUser, null); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + + // ugi for remote user + request = getMockRequest(realUser, null, null); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getShortUserName(), realUser); + checkUgiFromAuth(ugi); + + // ugi for remote user = real user + request = getMockRequest(realUser, realUser, null); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getShortUserName(), realUser); + checkUgiFromAuth(ugi); + + // ugi for remote user != real user + request = getMockRequest(realUser, user, null); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name="+user+" != expected="+realUser, + ioe.getMessage()); + } + } + + @Test + public void testGetProxyUgi() throws IOException { + conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/"); + ServletContext context = mock(ServletContext.class); + String realUser = "TheDoctor"; + String user = "TheNurse"; + conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + + conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".groups", "*"); + conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".hosts", "*"); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + UserGroupInformation.setConfiguration(conf); + UserGroupInformation ugi; + HttpServletRequest request; + + // have to be auth-ed with remote user + request = getMockRequest(null, null, user); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + request = getMockRequest(null, realUser, user); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Security enabled but user not authenticated by filter", + ioe.getMessage()); + } + + // proxy ugi for user via remote user + request = getMockRequest(realUser, null, user); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromAuth(ugi); + + // proxy ugi for user vi a remote user = real user + request = getMockRequest(realUser, realUser, user); + ugi = JspHelper.getUGI(context, request, conf); + Assert.assertNotNull(ugi.getRealUser()); + Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser); + Assert.assertEquals(ugi.getShortUserName(), user); + checkUgiFromAuth(ugi); + + // proxy ugi for user via remote user != real user + request = getMockRequest(realUser, user, user); + try { + JspHelper.getUGI(context, request, conf); + Assert.fail("bad request allowed"); + } catch (IOException ioe) { + Assert.assertEquals( + "Usernames not matched: name="+user+" != expected="+realUser, + ioe.getMessage()); + } + + // try to get get a proxy user with unauthorized user + try { + request = getMockRequest(user, null, realUser); + JspHelper.getUGI(context, request, conf); + Assert.fail("bad proxy request allowed"); + } catch (AuthorizationException ae) { + Assert.assertEquals( + "User: " + user + " is not allowed to impersonate " + realUser, + ae.getMessage()); + } + try { + request = getMockRequest(user, user, realUser); + JspHelper.getUGI(context, request, conf); + Assert.fail("bad proxy request allowed"); + } catch (AuthorizationException ae) { + Assert.assertEquals( + "User: " + user + " is not allowed to impersonate " + realUser, + ae.getMessage()); + } + } + + private HttpServletRequest getMockRequest(String remoteUser, String user, String doAs) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getParameter(UserParam.NAME)).thenReturn(user); + if (doAs != null) { + when(request.getParameter(DoAsParam.NAME)).thenReturn(doAs); + } + when(request.getRemoteUser()).thenReturn(remoteUser); + return request; + } + + private void checkUgiFromAuth(UserGroupInformation ugi) { + if (ugi.getRealUser() != null) { + Assert.assertEquals(AuthenticationMethod.PROXY, + ugi.getAuthenticationMethod()); + Assert.assertEquals(AuthenticationMethod.KERBEROS_SSL, + ugi.getRealUser().getAuthenticationMethod()); + } else { + Assert.assertEquals(AuthenticationMethod.KERBEROS_SSL, + ugi.getAuthenticationMethod()); + } + } + + private void checkUgiFromToken(UserGroupInformation ugi) { + if (ugi.getRealUser() != null) { + Assert.assertEquals(AuthenticationMethod.PROXY, + ugi.getAuthenticationMethod()); + Assert.assertEquals(AuthenticationMethod.TOKEN, + ugi.getRealUser().getAuthenticationMethod()); + } else { + Assert.assertEquals(AuthenticationMethod.TOKEN, + ugi.getAuthenticationMethod()); + } + } } From 0c4e670eff6a062561b254ae2b62f3900b37b37b Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 27 Jul 2012 21:51:07 +0000 Subject: [PATCH 30/39] HDFS-3679. fuse_dfs notrash option sets usetrash. Contributed by Conrad Meyer. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366545 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 5 ++++- .../hadoop-hdfs/src/main/native/fuse-dfs/fuse_options.c | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4c7d679ad67..1f9ce6f58d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -305,7 +305,8 @@ Branch-2 ( Unreleased changes ) HDFS-3613. GSet prints some INFO level values, which aren't really very useful to all (Andrew Wang via harsh) - HDFS-3611. NameNode prints unnecessary WARNs about edit log normally skipping a few bytes. (Colin Patrick McCabe via harsh) + HDFS-3611. NameNode prints unnecessary WARNs about edit log normally skipping + a few bytes. (Colin Patrick McCabe via harsh) HDFS-3582. Hook daemon process exit for testing. (eli) @@ -538,6 +539,8 @@ Branch-2 ( Unreleased changes ) HDFS-3626. Creating file with invalid path can corrupt edit log (todd) + HDFS-3679. fuse_dfs notrash option sets usetrash. (Conrad Meyer via suresh) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_options.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_options.c index 35829743041..8461ce40f91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_options.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_options.c @@ -124,7 +124,7 @@ int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs options.usetrash = 1; break; case KEY_NOTRASH: - options.usetrash = 1; + options.usetrash = 0; break; case KEY_RO: options.read_only = 1; From e4eec269d91ae541a321ae2f28ff03310682b3fe Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Sat, 28 Jul 2012 05:57:47 +0000 Subject: [PATCH 31/39] HDFS-3667. Add retry support to WebHdfsFileSystem. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1366601 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/ByteRangeInputStream.java | 18 +- .../apache/hadoop/hdfs/HftpFileSystem.java | 29 +- .../apache/hadoop/hdfs/NameNodeProxies.java | 11 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 316 ++++++++++++------ .../hdfs/web/resources/DeleteOpParam.java | 5 + .../hadoop/hdfs/web/resources/GetOpParam.java | 29 +- .../hdfs/web/resources/HttpOpParam.java | 34 +- .../hdfs/web/resources/PostOpParam.java | 5 + .../hadoop/hdfs/web/resources/PutOpParam.java | 13 +- .../hadoop/hdfs/TestDFSClientRetries.java | 60 +++- .../TestDelegationTokenForProxyUser.java | 9 +- .../hdfs/web/TestOffsetUrlInputStream.java | 73 ---- .../hadoop/hdfs/web/TestWebHdfsRetries.java | 37 ++ .../hadoop/hdfs/web/WebHdfsTestUtil.java | 10 +- 15 files changed, 396 insertions(+), 255 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsRetries.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1f9ce6f58d3..5b5a503c8f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -373,6 +373,8 @@ Branch-2 ( Unreleased changes ) HDFS-3697. Enable fadvise readahead by default. (todd) + HDFS-3667. Add retry support to WebHdfsFileSystem. (szetszwo) + BUG FIXES HDFS-3385. The last block of INodeFileUnderConstruction is not diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java index 62c67f9e9d0..e745714c9be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java @@ -57,9 +57,9 @@ public abstract class ByteRangeInputStream extends FSInputStream { return url; } - protected abstract HttpURLConnection openConnection() throws IOException; - - protected abstract HttpURLConnection openConnection(final long offset) throws IOException; + /** Connect to server with a data offset. */ + protected abstract HttpURLConnection connect(final long offset, + final boolean resolved) throws IOException; } enum StreamStatus { @@ -85,9 +85,6 @@ public abstract class ByteRangeInputStream extends FSInputStream { this.resolvedURL = r; } - protected abstract void checkResponseCode(final HttpURLConnection connection - ) throws IOException; - protected abstract URL getResolvedUrl(final HttpURLConnection connection ) throws IOException; @@ -113,13 +110,10 @@ public abstract class ByteRangeInputStream extends FSInputStream { protected InputStream openInputStream() throws IOException { // Use the original url if no resolved url exists, eg. if // it's the first time a request is made. - final URLOpener opener = - (resolvedURL.getURL() == null) ? originalURL : resolvedURL; - - final HttpURLConnection connection = opener.openConnection(startPos); - connection.connect(); - checkResponseCode(connection); + final boolean resolved = resolvedURL.getURL() != null; + final URLOpener opener = resolved? resolvedURL: originalURL; + final HttpURLConnection connection = opener.connect(startPos, resolved); final String cl = connection.getHeaderField(StreamFile.CONTENT_LENGTH); if (cl == null) { throw new IOException(StreamFile.CONTENT_LENGTH+" header is missing"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index 6df5f2dd9b0..8c73e2a6bee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -342,19 +342,28 @@ public class HftpFileSystem extends FileSystem super(url); } - @Override protected HttpURLConnection openConnection() throws IOException { return (HttpURLConnection)URLUtils.openConnection(url); } /** Use HTTP Range header for specifying offset. */ @Override - protected HttpURLConnection openConnection(final long offset) throws IOException { + protected HttpURLConnection connect(final long offset, + final boolean resolved) throws IOException { final HttpURLConnection conn = openConnection(); conn.setRequestMethod("GET"); if (offset != 0L) { conn.setRequestProperty("Range", "bytes=" + offset + "-"); } + conn.connect(); + + //Expects HTTP_OK or HTTP_PARTIAL response codes. + final int code = conn.getResponseCode(); + if (offset != 0L && code != HttpURLConnection.HTTP_PARTIAL) { + throw new IOException("HTTP_PARTIAL expected, received " + code); + } else if (offset == 0L && code != HttpURLConnection.HTTP_OK) { + throw new IOException("HTTP_OK expected, received " + code); + } return conn; } } @@ -368,22 +377,6 @@ public class HftpFileSystem extends FileSystem this(new RangeHeaderUrlOpener(url), new RangeHeaderUrlOpener(null)); } - /** Expects HTTP_OK and HTTP_PARTIAL response codes. */ - @Override - protected void checkResponseCode(final HttpURLConnection connection - ) throws IOException { - final int code = connection.getResponseCode(); - if (startPos != 0 && code != HttpURLConnection.HTTP_PARTIAL) { - // We asked for a byte range but did not receive a partial content - // response... - throw new IOException("HTTP_PARTIAL expected, received " + code); - } else if (startPos == 0 && code != HttpURLConnection.HTTP_OK) { - // We asked for all bytes from the beginning but didn't receive a 200 - // response (none of the other 2xx codes are valid here) - throw new IOException("HTTP_OK expected, received " + code); - } - } - @Override protected URL getResolvedUrl(final HttpURLConnection connection) { return connection.getURL(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index cc6517daa52..12ec985fb25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -259,7 +259,7 @@ public class NameNodeProxies { * * Note that dfs.client.retry.max < 0 is not allowed. */ - private static RetryPolicy getDefaultRpcRetryPolicy(Configuration conf) { + public static RetryPolicy getDefaultRetryPolicy(Configuration conf) { final RetryPolicy multipleLinearRandomRetry = getMultipleLinearRandomRetry(conf); if (LOG.isDebugEnabled()) { LOG.debug("multipleLinearRandomRetry = " + multipleLinearRandomRetry); @@ -300,6 +300,13 @@ public class NameNodeProxies { + p.getClass().getSimpleName() + ", exception=" + e); return p.shouldRetry(e, retries, failovers, isMethodIdempotent); } + + @Override + public String toString() { + return "RetryPolicy[" + multipleLinearRandomRetry + ", " + + RetryPolicies.TRY_ONCE_THEN_FAIL.getClass().getSimpleName() + + "]"; + } }; } } @@ -335,7 +342,7 @@ public class NameNodeProxies { boolean withRetries) throws IOException { RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class); - final RetryPolicy defaultPolicy = getDefaultRpcRetryPolicy(conf); + final RetryPolicy defaultPolicy = getDefaultRetryPolicy(conf); final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class); ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy( ClientNamenodeProtocolPB.class, version, address, ugi, conf, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 420d74d5829..44e70e8636a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -55,6 +55,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.ByteRangeInputStream; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; @@ -88,6 +89,7 @@ import org.apache.hadoop.hdfs.web.resources.ReplicationParam; import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam; import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; @@ -147,6 +149,7 @@ public class WebHdfsFileSystem extends FileSystem private URI uri; private Token delegationToken; private final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); + private RetryPolicy retryPolicy = null; private Path workingDir; { @@ -179,6 +182,7 @@ public class WebHdfsFileSystem extends FileSystem throw new IllegalArgumentException(e); } this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); + this.retryPolicy = NameNodeProxies.getDefaultRetryPolicy(conf); this.workingDir = getHomeDirectory(); if (UserGroupInformation.isSecurityEnabled()) { @@ -276,13 +280,13 @@ public class WebHdfsFileSystem extends FileSystem } private static Map validateResponse(final HttpOpParam.Op op, - final HttpURLConnection conn) throws IOException { + final HttpURLConnection conn, boolean unwrapException) throws IOException { final int code = conn.getResponseCode(); if (code != op.getExpectedHttpResponseCode()) { final Map m; try { m = jsonParse(conn, true); - } catch(IOException e) { + } catch(Exception e) { throw new IOException("Unexpected HTTP response: code=" + code + " != " + op.getExpectedHttpResponseCode() + ", " + op.toQueryString() + ", message=" + conn.getResponseMessage(), e); @@ -293,21 +297,42 @@ public class WebHdfsFileSystem extends FileSystem } final RemoteException re = JsonUtil.toRemoteException(m); - throw re.unwrapRemoteException(AccessControlException.class, - InvalidToken.class, - AuthenticationException.class, - AuthorizationException.class, - FileAlreadyExistsException.class, - FileNotFoundException.class, - ParentNotDirectoryException.class, - UnresolvedPathException.class, - SafeModeException.class, - DSQuotaExceededException.class, - NSQuotaExceededException.class); + throw unwrapException? toIOException(re): re; } return null; } + /** + * Covert an exception to an IOException. + * + * For a non-IOException, wrap it with IOException. + * For a RemoteException, unwrap it. + * For an IOException which is not a RemoteException, return it. + */ + private static IOException toIOException(Exception e) { + if (!(e instanceof IOException)) { + return new IOException(e); + } + + final IOException ioe = (IOException)e; + if (!(ioe instanceof RemoteException)) { + return ioe; + } + + final RemoteException re = (RemoteException)ioe; + return re.unwrapRemoteException(AccessControlException.class, + InvalidToken.class, + AuthenticationException.class, + AuthorizationException.class, + FileAlreadyExistsException.class, + FileNotFoundException.class, + ParentNotDirectoryException.class, + UnresolvedPathException.class, + SafeModeException.class, + DSQuotaExceededException.class, + NSQuotaExceededException.class); + } + /** * Return a URL pointing to given path on the namenode. * @@ -362,70 +387,15 @@ public class WebHdfsFileSystem extends FileSystem } private HttpURLConnection getHttpUrlConnection(URL url) - throws IOException { + throws IOException, AuthenticationException { final HttpURLConnection conn; - try { - if (ugi.hasKerberosCredentials()) { - conn = new AuthenticatedURL(AUTH).openConnection(url, authToken); - } else { - conn = (HttpURLConnection)url.openConnection(); - } - } catch (AuthenticationException e) { - throw new IOException("Authentication failed, url=" + url, e); + if (ugi.hasKerberosCredentials()) { + conn = new AuthenticatedURL(AUTH).openConnection(url, authToken); + } else { + conn = (HttpURLConnection)url.openConnection(); } return conn; } - - private HttpURLConnection httpConnect(final HttpOpParam.Op op, final Path fspath, - final Param... parameters) throws IOException { - final URL url = toUrl(op, fspath, parameters); - - //connect and get response - HttpURLConnection conn = getHttpUrlConnection(url); - try { - conn.setRequestMethod(op.getType().toString()); - if (op.getDoOutput()) { - conn = twoStepWrite(conn, op); - conn.setRequestProperty("Content-Type", "application/octet-stream"); - } - conn.setDoOutput(op.getDoOutput()); - conn.connect(); - return conn; - } catch (IOException e) { - conn.disconnect(); - throw e; - } - } - - /** - * Two-step Create/Append: - * Step 1) Submit a Http request with neither auto-redirect nor data. - * Step 2) Submit another Http request with the URL from the Location header with data. - * - * The reason of having two-step create/append is for preventing clients to - * send out the data before the redirect. This issue is addressed by the - * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3. - * Unfortunately, there are software library bugs (e.g. Jetty 6 http server - * and Java 6 http client), which do not correctly implement "Expect: - * 100-continue". The two-step create/append is a temporary workaround for - * the software library bugs. - */ - static HttpURLConnection twoStepWrite(HttpURLConnection conn, - final HttpOpParam.Op op) throws IOException { - //Step 1) Submit a Http request with neither auto-redirect nor data. - conn.setInstanceFollowRedirects(false); - conn.setDoOutput(false); - conn.connect(); - validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn); - final String redirect = conn.getHeaderField("Location"); - conn.disconnect(); - - //Step 2) Submit another Http request with the URL from the Location header with data. - conn = (HttpURLConnection)new URL(redirect).openConnection(); - conn.setRequestMethod(op.getType().toString()); - conn.setChunkedStreamingMode(32 << 10); //32kB-chunk - return conn; - } /** * Run a http operation. @@ -439,12 +409,158 @@ public class WebHdfsFileSystem extends FileSystem */ private Map run(final HttpOpParam.Op op, final Path fspath, final Param... parameters) throws IOException { - final HttpURLConnection conn = httpConnect(op, fspath, parameters); - try { - final Map m = validateResponse(op, conn); - return m != null? m: jsonParse(conn, false); - } finally { - conn.disconnect(); + return new Runner(op, fspath, parameters).run().json; + } + + /** + * This class is for initialing a HTTP connection, connecting to server, + * obtaining a response, and also handling retry on failures. + */ + class Runner { + private final HttpOpParam.Op op; + private final URL url; + private final boolean redirected; + + private boolean checkRetry; + private HttpURLConnection conn = null; + private Map json = null; + + Runner(final HttpOpParam.Op op, final URL url, final boolean redirected) { + this.op = op; + this.url = url; + this.redirected = redirected; + } + + Runner(final HttpOpParam.Op op, final Path fspath, + final Param... parameters) throws IOException { + this(op, toUrl(op, fspath, parameters), false); + } + + Runner(final HttpOpParam.Op op, final HttpURLConnection conn) { + this(op, null, false); + this.conn = conn; + } + + private void init() throws IOException { + checkRetry = !redirected; + try { + conn = getHttpUrlConnection(url); + } catch(AuthenticationException ae) { + checkRetry = false; + throw new IOException("Authentication failed, url=" + url, ae); + } + } + + private void connect() throws IOException { + connect(op.getDoOutput()); + } + + private void connect(boolean doOutput) throws IOException { + conn.setRequestMethod(op.getType().toString()); + conn.setDoOutput(doOutput); + conn.setInstanceFollowRedirects(false); + conn.connect(); + } + + private void disconnect() { + if (conn != null) { + conn.disconnect(); + conn = null; + } + } + + Runner run() throws IOException { + for(int retry = 0; ; retry++) { + try { + init(); + if (op.getDoOutput()) { + twoStepWrite(); + } else { + getResponse(op != GetOpParam.Op.OPEN); + } + return this; + } catch(IOException ioe) { + shouldRetry(ioe, retry); + } + } + } + + private void shouldRetry(final IOException ioe, final int retry + ) throws IOException { + if (checkRetry) { + try { + final RetryPolicy.RetryAction a = retryPolicy.shouldRetry( + ioe, retry, 0, true); + if (a.action == RetryPolicy.RetryAction.RetryDecision.RETRY) { + LOG.info("Retrying connect to namenode: " + nnAddr + + ". Already tried " + retry + " time(s); retry policy is " + + retryPolicy + ", delay " + a.delayMillis + "ms."); + Thread.sleep(a.delayMillis); + return; + } + } catch(Exception e) { + LOG.warn("Original exception is ", ioe); + throw toIOException(e); + } + } + throw toIOException(ioe); + } + + /** + * Two-step Create/Append: + * Step 1) Submit a Http request with neither auto-redirect nor data. + * Step 2) Submit another Http request with the URL from the Location header with data. + * + * The reason of having two-step create/append is for preventing clients to + * send out the data before the redirect. This issue is addressed by the + * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3. + * Unfortunately, there are software library bugs (e.g. Jetty 6 http server + * and Java 6 http client), which do not correctly implement "Expect: + * 100-continue". The two-step create/append is a temporary workaround for + * the software library bugs. + */ + HttpURLConnection twoStepWrite() throws IOException { + //Step 1) Submit a Http request with neither auto-redirect nor data. + connect(false); + validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn, false); + final String redirect = conn.getHeaderField("Location"); + disconnect(); + checkRetry = false; + + //Step 2) Submit another Http request with the URL from the Location header with data. + conn = (HttpURLConnection)new URL(redirect).openConnection(); + conn.setChunkedStreamingMode(32 << 10); //32kB-chunk + connect(); + return conn; + } + + FSDataOutputStream write(final int bufferSize) throws IOException { + return WebHdfsFileSystem.this.write(op, conn, bufferSize); + } + + void getResponse(boolean getJsonAndDisconnect) throws IOException { + try { + connect(); + if (!redirected && op.getRedirect()) { + final String redirect = conn.getHeaderField("Location"); + json = validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), + conn, false); + disconnect(); + + checkRetry = false; + conn = (HttpURLConnection)new URL(redirect).openConnection(); + connect(); + } + + json = validateResponse(op, conn, false); + if (json == null && getJsonAndDisconnect) { + json = jsonParse(conn, false); + } + } finally { + if (getJsonAndDisconnect) { + disconnect(); + } + } } } @@ -578,7 +694,7 @@ public class WebHdfsFileSystem extends FileSystem super.close(); } finally { try { - validateResponse(op, conn); + validateResponse(op, conn, true); } finally { conn.disconnect(); } @@ -594,13 +710,14 @@ public class WebHdfsFileSystem extends FileSystem statistics.incrementWriteOps(1); final HttpOpParam.Op op = PutOpParam.Op.CREATE; - final HttpURLConnection conn = httpConnect(op, f, + return new Runner(op, f, new PermissionParam(applyUMask(permission)), new OverwriteParam(overwrite), new BufferSizeParam(bufferSize), new ReplicationParam(replication), - new BlockSizeParam(blockSize)); - return write(op, conn, bufferSize); + new BlockSizeParam(blockSize)) + .run() + .write(bufferSize); } @Override @@ -609,9 +726,9 @@ public class WebHdfsFileSystem extends FileSystem statistics.incrementWriteOps(1); final HttpOpParam.Op op = PostOpParam.Op.APPEND; - final HttpURLConnection conn = httpConnect(op, f, - new BufferSizeParam(bufferSize)); - return write(op, conn, bufferSize); + return new Runner(op, f, new BufferSizeParam(bufferSize)) + .run() + .write(bufferSize); } @SuppressWarnings("deprecation") @@ -638,26 +755,17 @@ public class WebHdfsFileSystem extends FileSystem } class OffsetUrlOpener extends ByteRangeInputStream.URLOpener { - /** The url with offset parameter */ - private URL offsetUrl; - OffsetUrlOpener(final URL url) { super(url); } - /** Open connection with offset url. */ + /** Setup offset url and connect. */ @Override - protected HttpURLConnection openConnection() throws IOException { - return getHttpUrlConnection(offsetUrl); - } - - /** Setup offset url before open connection. */ - @Override - protected HttpURLConnection openConnection(final long offset) throws IOException { - offsetUrl = offset == 0L? url: new URL(url + "&" + new OffsetParam(offset)); - final HttpURLConnection conn = openConnection(); - conn.setRequestMethod("GET"); - return conn; + protected HttpURLConnection connect(final long offset, + final boolean resolved) throws IOException { + final URL offsetUrl = offset == 0L? url + : new URL(url + "&" + new OffsetParam(offset)); + return new Runner(GetOpParam.Op.OPEN, offsetUrl, resolved).run().conn; } } @@ -698,12 +806,6 @@ public class WebHdfsFileSystem extends FileSystem OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) { super(o, r); } - - @Override - protected void checkResponseCode(final HttpURLConnection connection - ) throws IOException { - validateResponse(GetOpParam.Op.OPEN, connection); - } /** Remove offset parameter before returning the resolved url. */ @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java index 12962b4a4ee..a82b8a72c8e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java @@ -43,6 +43,11 @@ public class DeleteOpParam extends HttpOpParam { return false; } + @Override + public boolean getRedirect() { + return false; + } + @Override public int getExpectedHttpResponseCode() { return expectedHttpResponseCode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 34d6e12ecf9..6bbd9e21693 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -23,25 +23,27 @@ import java.net.HttpURLConnection; public class GetOpParam extends HttpOpParam { /** Get operations. */ public static enum Op implements HttpOpParam.Op { - OPEN(HttpURLConnection.HTTP_OK), + OPEN(true, HttpURLConnection.HTTP_OK), - GETFILESTATUS(HttpURLConnection.HTTP_OK), - LISTSTATUS(HttpURLConnection.HTTP_OK), - GETCONTENTSUMMARY(HttpURLConnection.HTTP_OK), - GETFILECHECKSUM(HttpURLConnection.HTTP_OK), + GETFILESTATUS(false, HttpURLConnection.HTTP_OK), + LISTSTATUS(false, HttpURLConnection.HTTP_OK), + GETCONTENTSUMMARY(false, HttpURLConnection.HTTP_OK), + GETFILECHECKSUM(true, HttpURLConnection.HTTP_OK), - GETHOMEDIRECTORY(HttpURLConnection.HTTP_OK), - GETDELEGATIONTOKEN(HttpURLConnection.HTTP_OK), - GETDELEGATIONTOKENS(HttpURLConnection.HTTP_OK), + GETHOMEDIRECTORY(false, HttpURLConnection.HTTP_OK), + GETDELEGATIONTOKEN(false, HttpURLConnection.HTTP_OK), + GETDELEGATIONTOKENS(false, HttpURLConnection.HTTP_OK), /** GET_BLOCK_LOCATIONS is a private unstable op. */ - GET_BLOCK_LOCATIONS(HttpURLConnection.HTTP_OK), + GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK), - NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED); + NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED); + final boolean redirect; final int expectedHttpResponseCode; - Op(final int expectedHttpResponseCode) { + Op(final boolean redirect, final int expectedHttpResponseCode) { + this.redirect = redirect; this.expectedHttpResponseCode = expectedHttpResponseCode; } @@ -55,6 +57,11 @@ public class GetOpParam extends HttpOpParam { return false; } + @Override + public boolean getRedirect() { + return redirect; + } + @Override public int getExpectedHttpResponseCode() { return expectedHttpResponseCode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java index ab32ab59aa3..9765f67996c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs.web.resources; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import javax.ws.rs.core.Response; @@ -42,6 +46,9 @@ public abstract class HttpOpParam & HttpOpParam.Op> /** @return true if the operation will do output. */ public boolean getDoOutput(); + /** @return true if the operation will be redirected. */ + public boolean getRedirect(); + /** @return true the expected http response code. */ public int getExpectedHttpResponseCode(); @@ -51,15 +58,25 @@ public abstract class HttpOpParam & HttpOpParam.Op> /** Expects HTTP response 307 "Temporary Redirect". */ public static class TemporaryRedirectOp implements Op { - static final TemporaryRedirectOp CREATE = new TemporaryRedirectOp(PutOpParam.Op.CREATE); - static final TemporaryRedirectOp APPEND = new TemporaryRedirectOp(PostOpParam.Op.APPEND); + static final TemporaryRedirectOp CREATE = new TemporaryRedirectOp( + PutOpParam.Op.CREATE); + static final TemporaryRedirectOp APPEND = new TemporaryRedirectOp( + PostOpParam.Op.APPEND); + static final TemporaryRedirectOp OPEN = new TemporaryRedirectOp( + GetOpParam.Op.OPEN); + static final TemporaryRedirectOp GETFILECHECKSUM = new TemporaryRedirectOp( + GetOpParam.Op.GETFILECHECKSUM); + static final List values + = Collections.unmodifiableList(Arrays.asList( + new TemporaryRedirectOp[]{CREATE, APPEND, OPEN, GETFILECHECKSUM})); + /** Get an object for the given op. */ public static TemporaryRedirectOp valueOf(final Op op) { - if (op == CREATE.op) { - return CREATE; - } else if (op == APPEND.op) { - return APPEND; + for(TemporaryRedirectOp t : values) { + if (op == t.op) { + return t; + } } throw new IllegalArgumentException(op + " not found."); } @@ -80,6 +97,11 @@ public abstract class HttpOpParam & HttpOpParam.Op> return op.getDoOutput(); } + @Override + public boolean getRedirect() { + return false; + } + /** Override the original expected response with "Temporary Redirect". */ @Override public int getExpectedHttpResponseCode() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java index 89c02fe0d79..83f2cd19155 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java @@ -43,6 +43,11 @@ public class PostOpParam extends HttpOpParam { return true; } + @Override + public boolean getRedirect() { + return true; + } + @Override public int getExpectedHttpResponseCode() { return expectedHttpResponseCode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java index b6fc9198801..77bad214225 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java @@ -39,11 +39,11 @@ public class PutOpParam extends HttpOpParam { NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED); - final boolean doOutput; + final boolean doOutputAndRedirect; final int expectedHttpResponseCode; - Op(final boolean doOutput, final int expectedHttpResponseCode) { - this.doOutput = doOutput; + Op(final boolean doOutputAndRedirect, final int expectedHttpResponseCode) { + this.doOutputAndRedirect = doOutputAndRedirect; this.expectedHttpResponseCode = expectedHttpResponseCode; } @@ -54,7 +54,12 @@ public class PutOpParam extends HttpOpParam { @Override public boolean getDoOutput() { - return doOutput; + return doOutputAndRedirect; + } + + @Override + public boolean getRedirect() { + return doOutputAndRedirect; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 5c5de9db5ec..03ba1e96ba5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -47,6 +47,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; @@ -66,6 +67,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; @@ -74,6 +76,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; @@ -825,13 +828,17 @@ public class TestDFSClientRetries { /** Test client retry with namenode restarting. */ @Test public void testNamenodeRestart() throws Exception { + namenodeRestartTest(new Configuration(), false); + } + + public static void namenodeRestartTest(final Configuration conf, + final boolean isWebHDFS) throws Exception { ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); final List exceptions = new ArrayList(); final Path dir = new Path("/testNamenodeRestart"); - final Configuration conf = new Configuration(); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, true); final short numDatanodes = 3; @@ -841,16 +848,18 @@ public class TestDFSClientRetries { try { cluster.waitActive(); final DistributedFileSystem dfs = cluster.getFileSystem(); + final FileSystem fs = isWebHDFS? + WebHdfsTestUtil.getWebHdfsFileSystem(conf): dfs; final URI uri = dfs.getUri(); assertTrue(HdfsUtils.isHealthy(uri)); //create a file final long length = 1L << 20; final Path file1 = new Path(dir, "foo"); - DFSTestUtil.createFile(dfs, file1, length, numDatanodes, 20120406L); + DFSTestUtil.createFile(fs, file1, length, numDatanodes, 20120406L); //get file status - final FileStatus s1 = dfs.getFileStatus(file1); + final FileStatus s1 = fs.getFileStatus(file1); assertEquals(length, s1.getLen()); //shutdown namenode @@ -858,6 +867,25 @@ public class TestDFSClientRetries { cluster.shutdownNameNode(0); assertFalse(HdfsUtils.isHealthy(uri)); + //namenode is down, read the file in a thread + final Thread reader = new Thread(new Runnable() { + @Override + public void run() { + try { + //it should retry till namenode is up. + final FileSystem fs = createFsWithDifferentUsername(conf, isWebHDFS); + final FSDataInputStream in = fs.open(file1); + int count = 0; + for(; in.read() != -1; count++); + in.close(); + assertEquals(s1.getLen(), count); + } catch (Exception e) { + exceptions.add(e); + } + } + }); + reader.start(); + //namenode is down, create another file in a thread final Path file3 = new Path(dir, "file"); final Thread thread = new Thread(new Runnable() { @@ -865,7 +893,7 @@ public class TestDFSClientRetries { public void run() { try { //it should retry till namenode is up. - final FileSystem fs = AppendTestUtil.createHdfsWithDifferentUsername(conf); + final FileSystem fs = createFsWithDifferentUsername(conf, isWebHDFS); DFSTestUtil.createFile(fs, file3, length, numDatanodes, 20120406L); } catch (Exception e) { exceptions.add(e); @@ -892,12 +920,15 @@ public class TestDFSClientRetries { }).start(); //namenode is down, it should retry until namenode is up again. - final FileStatus s2 = dfs.getFileStatus(file1); + final FileStatus s2 = fs.getFileStatus(file1); assertEquals(s1, s2); //check file1 and file3 thread.join(); - assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file3)); + assertEquals(s1.getLen(), fs.getFileStatus(file3).getLen()); + assertEquals(fs.getFileChecksum(file1), fs.getFileChecksum(file3)); + + reader.join(); //enter safe mode assertTrue(HdfsUtils.isHealthy(uri)); @@ -922,8 +953,8 @@ public class TestDFSClientRetries { //namenode is in safe mode, create should retry until it leaves safe mode. final Path file2 = new Path(dir, "bar"); - DFSTestUtil.createFile(dfs, file2, length, numDatanodes, 20120406L); - assertEquals(dfs.getFileChecksum(file1), dfs.getFileChecksum(file2)); + DFSTestUtil.createFile(fs, file2, length, numDatanodes, 20120406L); + assertEquals(fs.getFileChecksum(file1), fs.getFileChecksum(file2)); assertTrue(HdfsUtils.isHealthy(uri)); @@ -931,7 +962,7 @@ public class TestDFSClientRetries { final Path nonExisting = new Path(dir, "nonExisting"); LOG.info("setPermission: " + nonExisting); try { - dfs.setPermission(nonExisting, new FsPermission((short)0)); + fs.setPermission(nonExisting, new FsPermission((short)0)); fail(); } catch(FileNotFoundException fnfe) { LOG.info("GOOD!", fnfe); @@ -949,6 +980,17 @@ public class TestDFSClientRetries { } } + public static FileSystem createFsWithDifferentUsername( + final Configuration conf, final boolean isWebHDFS + ) throws IOException, InterruptedException { + String username = UserGroupInformation.getCurrentUser().getShortUserName()+"_XXX"; + UserGroupInformation ugi = + UserGroupInformation.createUserForTesting(username, new String[]{"supergroup"}); + + return isWebHDFS? WebHdfsTestUtil.getWebHdfsFileSystemAs(ugi, conf) + : DFSTestUtil.getFileSystemAs(ugi, conf); + } + @Test public void testMultipleLinearRandomRetry() { parseMultipleLinearRandomRetry(null, ""); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java index 02c04141050..fe32fcdfbe5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java @@ -44,7 +44,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -140,9 +139,7 @@ public class TestDelegationTokenForProxyUser { .doAs(new PrivilegedExceptionAction>() { @Override public Token run() throws IOException { - DistributedFileSystem dfs = (DistributedFileSystem) cluster - .getFileSystem(); - return dfs.getDelegationToken("RenewerUser"); + return cluster.getFileSystem().getDelegationToken("RenewerUser"); } }); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); @@ -206,7 +203,7 @@ public class TestDelegationTokenForProxyUser { final PutOpParam.Op op = PutOpParam.Op.CREATE; final URL url = WebHdfsTestUtil.toUrl(webhdfs, op, f, new DoAsParam(PROXY_USER)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn = WebHdfsTestUtil.twoStepWrite(conn, op); + conn = WebHdfsTestUtil.twoStepWrite(webhdfs, op, conn); final FSDataOutputStream out = WebHdfsTestUtil.write(webhdfs, op, conn, 4096); out.write("Hello, webhdfs user!".getBytes()); out.close(); @@ -221,7 +218,7 @@ public class TestDelegationTokenForProxyUser { final PostOpParam.Op op = PostOpParam.Op.APPEND; final URL url = WebHdfsTestUtil.toUrl(webhdfs, op, f, new DoAsParam(PROXY_USER)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn = WebHdfsTestUtil.twoStepWrite(conn, op); + conn = WebHdfsTestUtil.twoStepWrite(webhdfs, op, conn); final FSDataOutputStream out = WebHdfsTestUtil.write(webhdfs, op, conn, 4096); out.write("\nHello again!".getBytes()); out.close(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestOffsetUrlInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestOffsetUrlInputStream.java index 4ef0dd680e7..1a95fc8c327 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestOffsetUrlInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestOffsetUrlInputStream.java @@ -18,22 +18,10 @@ package org.apache.hadoop.hdfs.web; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import java.io.IOException; -import java.net.URI; import java.net.URL; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.TestByteRangeInputStream.MockHttpURLConnection; -import org.apache.hadoop.hdfs.web.WebHdfsFileSystem.OffsetUrlInputStream; -import org.apache.hadoop.hdfs.web.WebHdfsFileSystem.OffsetUrlOpener; import org.junit.Test; public class TestOffsetUrlInputStream { @@ -73,65 +61,4 @@ public class TestOffsetUrlInputStream { WebHdfsFileSystem.removeOffsetParam(new URL(s)).toString()); } } - - @Test - public void testByteRange() throws Exception { - final Configuration conf = new Configuration(); - final String uri = WebHdfsFileSystem.SCHEME + "://localhost:50070/"; - final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)FileSystem.get(new URI(uri), conf); - - OffsetUrlOpener ospy = spy(webhdfs.new OffsetUrlOpener(new URL("http://test/"))); - doReturn(new MockHttpURLConnection(ospy.getURL())).when(ospy) - .openConnection(); - OffsetUrlOpener rspy = spy(webhdfs.new OffsetUrlOpener((URL) null)); - doReturn(new MockHttpURLConnection(rspy.getURL())).when(rspy) - .openConnection(); - final OffsetUrlInputStream is = new OffsetUrlInputStream(ospy, rspy); - - assertEquals("getPos wrong", 0, is.getPos()); - - is.read(); - - assertNull("Initial call made incorrectly (Range Check)", ospy - .openConnection().getRequestProperty("Range")); - - assertEquals("getPos should be 1 after reading one byte", 1, is.getPos()); - - is.read(); - - assertEquals("getPos should be 2 after reading two bytes", 2, is.getPos()); - - // No additional connections should have been made (no seek) - - rspy.setURL(new URL("http://resolvedurl/")); - - is.seek(100); - is.read(); - - assertEquals("getPos should be 101 after reading one byte", 101, - is.getPos()); - - verify(rspy, times(1)).openConnection(); - - is.seek(101); - is.read(); - - verify(rspy, times(1)).openConnection(); - - // Seek to 101 should not result in another request" - - is.seek(2500); - is.read(); - - ((MockHttpURLConnection) rspy.openConnection()).setResponseCode(206); - is.seek(0); - - try { - is.read(); - fail("Exception should be thrown when 206 response is given " - + "but 200 is expected"); - } catch (IOException e) { - WebHdfsFileSystem.LOG.info(e.toString()); - } - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsRetries.java new file mode 100644 index 00000000000..27ff56d8395 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsRetries.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.web; + +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.TestDFSClientRetries; +import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; +import org.apache.log4j.Level; +import org.junit.Test; + +/** Test WebHdfsFileSystem retry on failures. */ +public class TestWebHdfsRetries { + /** Test client retry with namenode restarting. */ + @Test + public void testNamenodeRestart() throws Exception { + ((Log4JLogger)NamenodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL); + final Configuration conf = WebHdfsTestUtil.createConf(); + TestDFSClientRetries.namenodeRestartTest(conf, true); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java index 9ae0fb28c21..e51a2524ecd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java @@ -79,13 +79,9 @@ public class WebHdfsTestUtil { return WebHdfsFileSystem.jsonParse(conn, false); } - public static HttpURLConnection twoStepWrite(HttpURLConnection conn, - final HttpOpParam.Op op) throws IOException { - conn.setRequestMethod(op.getType().toString()); - conn = WebHdfsFileSystem.twoStepWrite(conn, op); - conn.setDoOutput(true); - conn.connect(); - return conn; + public static HttpURLConnection twoStepWrite(final WebHdfsFileSystem webhdfs, + final HttpOpParam.Op op, HttpURLConnection conn) throws IOException { + return webhdfs.new Runner(op, conn).twoStepWrite(); } public static FSDataOutputStream write(final WebHdfsFileSystem webhdfs, From 98f1523b20603f7013baeb697662c3bb37505cc6 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 30 Jul 2012 15:18:11 +0000 Subject: [PATCH 32/39] HADOOP-8627. FS deleteOnExit may delete the wrong path (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367114 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../apache/hadoop/fs/FilterFileSystem.java | 17 --------- .../hadoop/fs/TestFilterFileSystem.java | 4 +- .../fs/viewfs/TestChRootedFileSystem.java | 37 ++++++++++++++++++- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f01b8a5db2e..3d1d755e4b6 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -857,6 +857,8 @@ Release 0.23.3 - UNRELEASED HADOOP-8613. AbstractDelegationTokenIdentifier#getUser() should set token auth type. (daryn) + HADOOP-8627. FS deleteOnExit may delete the wrong path (daryn via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 6cbaf591e5a..38ddb6c5f58 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -191,23 +191,6 @@ public class FilterFileSystem extends FileSystem { return fs.delete(f, recursive); } - /** - * Mark a path to be deleted when FileSystem is closed. - * When the JVM shuts down, - * all FileSystem objects will be closed automatically. - * Then, - * the marked path will be deleted as a result of closing the FileSystem. - * - * The path has to exist in the file system. - * - * @param f the path to delete. - * @return true if deleteOnExit is successful, otherwise false. - * @throws IOException - */ - public boolean deleteOnExit(Path f) throws IOException { - return fs.deleteOnExit(f); - } - /** List files in a directory. */ public FileStatus[] listStatus(Path f) throws IOException { return fs.listStatus(f); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index 647a583d103..a374c08a186 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -179,7 +179,9 @@ public class TestFilterFileSystem { public Token getDelegationToken(String renewer) throws IOException { return null; } - + public boolean deleteOnExit(Path f) throws IOException { + return false; + } public String getScheme() { return "dontcheck"; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java index 127866be1b5..44d7a4a7c13 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.viewfs.ChRootedFileSystem; @@ -33,6 +34,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.mockito.Mockito.*; public class TestChRootedFileSystem { FileSystem fSys; // The ChRoootedFs @@ -314,4 +316,37 @@ public class TestChRootedFileSystem { public void testResolvePathNonExisting() throws IOException { fSys.resolvePath(new Path("/nonExisting")); } -} + + @Test + public void testDeleteOnExitPathHandling() throws IOException { + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI chrootUri = URI.create("mockfs://foo/a/b"); + ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); + FileSystem mockFs = ((FilterFileSystem)chrootFs.getRawFileSystem()) + .getRawFileSystem(); + + // ensure delete propagates the correct path + Path chrootPath = new Path("/c"); + Path rawPath = new Path("/a/b/c"); + chrootFs.delete(chrootPath, false); + verify(mockFs).delete(eq(rawPath), eq(false)); + reset(mockFs); + + // fake that the path exists for deleteOnExit + FileStatus stat = mock(FileStatus.class); + when(mockFs.getFileStatus(eq(rawPath))).thenReturn(stat); + // ensure deleteOnExit propagates the correct path + chrootFs.deleteOnExit(chrootPath); + chrootFs.close(); + verify(mockFs).delete(eq(rawPath), eq(true)); + } + + static class MockFileSystem extends FilterFileSystem { + MockFileSystem() { + super(mock(FileSystem.class)); + } + public void initialize(URI name, Configuration conf) throws IOException {} + } +} \ No newline at end of file From d8584fde03c4b557ca6de74a0e32e6a4664039d9 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 30 Jul 2012 18:05:38 +0000 Subject: [PATCH 33/39] HADOOP-8634. Ensure FileSystem#close doesn't squawk for deleteOnExit paths (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367196 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/fs/FileSystem.java | 4 ++- .../hadoop/fs/TestFileSystemCaching.java | 28 +++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3d1d755e4b6..a2f3ff80e5c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -859,6 +859,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8627. FS deleteOnExit may delete the wrong path (daryn via bobby) + HADOOP-8634. Ensure FileSystem#close doesn't squawk for deleteOnExit paths + (daryn via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index f65d12c711c..42b30c4999b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -1224,7 +1224,9 @@ public abstract class FileSystem extends Configured implements Closeable { for (Iterator iter = deleteOnExit.iterator(); iter.hasNext();) { Path path = iter.next(); try { - delete(path, true); + if (exists(path)) { + delete(path, true); + } } catch (IOException e) { LOG.info("Ignoring failure to deleteOnExit for path " + path); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java index 910139b56ec..e8debf8dfaf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java @@ -35,7 +35,7 @@ import java.security.PrivilegedExceptionAction; import java.util.concurrent.Semaphore; import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.*; public class TestFileSystemCaching { @@ -267,4 +267,28 @@ public class TestFileSystemCaching { }); assertNotSame(fsA, fsA1); } -} + + @Test + public void testDeleteOnExitChecksExists() throws Exception { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path p = new Path("/a"); + + // path has to exist for deleteOnExit to register it + when(mockFs.getFileStatus(p)).thenReturn(new FileStatus()); + fs.deleteOnExit(p); + verify(mockFs).getFileStatus(eq(p)); + fs.close(); + verify(mockFs).delete(eq(p), anyBoolean()); + reset(mockFs); + + // make sure it doesn't try to delete a file that doesn't exist + when(mockFs.getFileStatus(p)).thenReturn(new FileStatus()); + fs.deleteOnExit(p); + verify(mockFs).getFileStatus(eq(p)); + reset(mockFs); + fs.close(); + verify(mockFs).getFileStatus(eq(p)); + verify(mockFs, never()).delete(any(Path.class), anyBoolean()); + } +} \ No newline at end of file From aeade62b95b4a39ea9f534c805fa1c6786a30077 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Mon, 30 Jul 2012 18:07:07 +0000 Subject: [PATCH 34/39] HDFS-3732. fuse_dfs: incorrect configuration value checked for connection expiry timer period. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367199 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5b5a503c8f5..05335ff7069 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -543,6 +543,9 @@ Branch-2 ( Unreleased changes ) HDFS-3679. fuse_dfs notrash option sets usetrash. (Conrad Meyer via suresh) + HDFS-3732. fuse_dfs: incorrect configuration value checked for connection + expiry timer period. (Colin Patrick McCabe via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c index c6624fad999..2a39d85263d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c @@ -135,7 +135,7 @@ int fuseConnectInit(const char *nnUri, int port) int ret; gTimerPeriod = FUSE_CONN_DEFAULT_TIMER_PERIOD; - ret = hdfsConfGetInt(HADOOP_FUSE_CONNECTION_TIMEOUT, &gTimerPeriod); + ret = hdfsConfGetInt(HADOOP_FUSE_TIMER_PERIOD, &gTimerPeriod); if (ret) { fprintf(stderr, "Unable to determine the configured value for %s.", HADOOP_FUSE_TIMER_PERIOD); From 48a2e2b4e279800b4371e6715e942392aa991ffe Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 30 Jul 2012 18:11:32 +0000 Subject: [PATCH 35/39] HADOOP-8550. hadoop fs -touchz automatically created parent directories (John George via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367202 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/shell/Touchz.java | 4 + .../org/apache/hadoop/hdfs/TestDFSShell.java | 13 ++++ .../src/test/resources/testHDFSConf.xml | 77 ++++++++++++++++++- 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a2f3ff80e5c..d1faef370a6 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -862,6 +862,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8634. Ensure FileSystem#close doesn't squawk for deleteOnExit paths (daryn via bobby) + HADOOP-8550. hadoop fs -touchz automatically created parent directories + (John George via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java index 18c9aa74c29..9e0844e0f41 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; +import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; /** * Unix touch like commands @@ -70,6 +71,9 @@ class Touch extends FsCommand { @Override protected void processNonexistentPath(PathData item) throws IOException { + if (!item.parentExists()) { + throw new PathNotFoundException(item.toString()); + } touchz(item); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index ce402b1878b..4387fb75a8c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -1108,6 +1108,19 @@ public class TestDFSShell { } assertEquals(0, val); + args = new String[2]; + args[0] = "-touchz"; + args[1] = "/test/mkdirs/thisDirNotExists/noFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + + args = new String[3]; args[0] = "-test"; args[1] = "-e"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index a611c2e3c39..40e3d4f919a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -55,6 +55,7 @@ ls: file using relative path + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -ls file1 @@ -76,6 +77,7 @@ ls: files using globbing + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 -fs NAMENODE -touchz file3 @@ -937,6 +939,7 @@ -fs NAMENODE -mkdir /dir0 -fs NAMENODE -mkdir /dir0/\* -fs NAMENODE -touchz /dir0/\*/file + -fs NAMENODE -mkdir /dir0/dir1/ -fs NAMENODE -touchz /dir0/dir1/file1 -fs NAMENODE -rm -r /dir0/\* -fs NAMENODE -ls -R /dir0 @@ -977,7 +980,7 @@ du: file using relative path - -fs NAMENODE -touchz test + -fs NAMENODE -mkdir -p dir -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytesZZ -fs NAMENODE -du data15bytesZZ @@ -995,6 +998,7 @@ du: files using globbing + -fs NAMENODE -mkdir -p CLITEST_DATA -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes data30bytes -fs NAMENODE -put CLITEST_DATA/data60bytes data60bytes @@ -1724,6 +1728,7 @@ mv: file (relative) to file (relative) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -mv file1 file2 -fs NAMENODE -ls file* @@ -2127,6 +2132,7 @@ cp: file (absolute path) to file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz /file1 -fs NAMENODE -cp /file1 file2 -fs NAMENODE -ls /file1 file2 @@ -2149,6 +2155,7 @@ cp: file (relative path) to file (absolute path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -cp file1 /file2 -fs NAMENODE -ls file1 /file2 @@ -2171,6 +2178,7 @@ cp: file (relative path) to file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -cp file1 file2 -fs NAMENODE -ls file1 file2 @@ -2720,6 +2728,7 @@ cp: putting file into an already existing destination with -f option(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -cp -f CLITEST_DATA/data120bytes /user/file0 -fs NAMENODE -cat /user/file0 @@ -3160,6 +3169,7 @@ rm: removing a file (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -rm /dir0/file0 @@ -3177,6 +3187,7 @@ rm: removing a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -rm file0 @@ -3194,6 +3205,7 @@ rm: removing files by globbing (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -touchz /dir0/file1 -fs NAMENODE -touchz /dir0/file2 @@ -3226,6 +3238,7 @@ rm: removing files by globbing (relative path) + -fs NAMENODE -mkdir dir -fs NAMENODE -touchz file0 -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 @@ -3555,6 +3568,7 @@ rm: removing a file (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -rm -r /dir0/file0 @@ -3572,6 +3586,7 @@ rm: removing a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -rm -r file0 @@ -3968,6 +3983,7 @@ put: putting file into a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -du data15bytes @@ -4066,6 +4082,7 @@ put: putting non existent file(absolute path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put /user/wrongdata file @@ -4083,6 +4100,7 @@ put: putting non existent file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put wrongdata file @@ -4100,6 +4118,7 @@ put: putting file into an already existing destination(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -put CLITEST_DATA/data15bytes /user/file0 @@ -4117,6 +4136,7 @@ put: putting file into an already existing destination with -f option(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -put -f CLITEST_DATA/data120bytes /user/file0 -fs NAMENODE -cat /user/file0 @@ -4135,6 +4155,7 @@ put: putting file into an already existing destination(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -put CLITEST_DATA/data15bytes file0 @@ -4154,6 +4175,7 @@ -fs NAMENODE -put CLITEST_DATA/data15bytes /data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes /data30bytes + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -put /data15bytes /data30bytes file0 @@ -4262,6 +4284,7 @@ put: putting local file into an already existing destination in hdfs:// path + -fs NAMENODE -mkdir hdfs:///user/ -fs NAMENODE -touchz hdfs:///user/file0 -fs NAMENODE -put CLITEST_DATA/data15bytes hdfs:///user/file0 @@ -4385,6 +4408,7 @@ put: putting local file into an already existing destination in Namenode's path + -fs NAMENODE -mkdir NAMENODE/user/ -fs NAMENODE -touchz NAMENODE/user/file0 -fs NAMENODE -put CLITEST_DATA/data15bytes NAMENODE/user/file0 @@ -4453,6 +4477,7 @@ copyFromLocal: copying file into a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -du data15bytes @@ -4551,6 +4576,7 @@ copyFromLocal: copying non existent file(absolute path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -copyFromLocal /user/wrongdata file @@ -4568,6 +4594,7 @@ copyFromLocal: copying non existent file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -copyFromLocal wrongdata file @@ -4585,6 +4612,7 @@ copyFromLocal: copying file into an already existing destination(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes /user/file0 @@ -4602,6 +4630,7 @@ copyFromLocal: copying file into an already existing destination with -f option(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -copyFromLocal -f CLITEST_DATA/data120bytes /user/file0 -fs NAMENODE -cat /user/file0 @@ -4621,6 +4650,7 @@ copyFromLocal: copying file into an already existing destination(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes file0 @@ -4640,6 +4670,7 @@ -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes /data15bytes -fs NAMENODE -copyFromLocal CLITEST_DATA/data30bytes /data30bytes + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -copyFromLocal /data15bytes /data30bytes file0 @@ -4750,6 +4781,7 @@ copyFromLocal: Test for hdfs:// path - copying local file into an already existing destination + -fs NAMENODE -mkdir hdfs:///user/ -fs NAMENODE -touchz hdfs:///user/file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes hdfs:///user/file0 @@ -4876,6 +4908,7 @@ copyFromLocal: Test for Namenode's path - copying local file into an already existing destination + -fs NAMENODE -mkdir NAMENODE/user/ -fs NAMENODE -touchz NAMENODE/user/file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes NAMENODE/user/file0 @@ -5012,6 +5045,7 @@ cat: contents of file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -cat data15bytes @@ -5796,6 +5830,7 @@ setrep: existent file (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -setrep 2 /dir0/file0 @@ -5813,6 +5848,7 @@ setrep: existent file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -setrep 2 file0 @@ -5830,6 +5866,7 @@ setrep: existent directory (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -touchz /dir0/file1 -fs NAMENODE -setrep -R 2 /dir0 @@ -5852,6 +5889,7 @@ setrep: existent directory (relative path) + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/file1 -fs NAMENODE -setrep -R 2 dir0 @@ -5906,6 +5944,7 @@ setrep: Test for hdfs:// path - existent file + -fs NAMENODE -mkdir hdfs:///dir0/ -fs NAMENODE -touchz hdfs:///dir0/file0 -fs NAMENODE -setrep 2 hdfs:///dir0/file0 @@ -5923,6 +5962,7 @@ setrep: Test for hdfs:// path - existent directory + -fs NAMENODE -mkdir hdfs:///dir0/ -fs NAMENODE -touchz hdfs:///dir0/file0 -fs NAMENODE -touchz hdfs:///dir0/file1 -fs NAMENODE -setrep -R 2 hdfs:///dir0 @@ -5945,6 +5985,7 @@ setrep: Test for hdfs:// path - non existent file + -fs NAMENODE -mkdir hdfs:///dir0/ -fs NAMENODE -setrep 2 hdfs:///dir0/file @@ -5961,6 +6002,7 @@ setrep: Test for Namenode's path - existent file + -fs NAMENODE -mkdir NAMENODE/dir0/ -fs NAMENODE -touchz NAMENODE/dir0/file0 -fs NAMENODE -setrep 2 NAMENODE/dir0/file0 @@ -5978,6 +6020,7 @@ setrep: Test for Namenode's path - existent directory + -fs NAMENODE -mkdir -p NAMENODE/dir0 -fs NAMENODE -touchz NAMENODE/dir0/file0 -fs NAMENODE -touchz NAMENODE/dir0/file1 -fs NAMENODE -setrep -R 2 NAMENODE/dir0 @@ -6017,6 +6060,7 @@ touchz: touching file (absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -du /user/file0 @@ -6030,10 +6074,26 @@ - + + + touchz: touching file in non-existent directory + + -fs NAMENODE -touchz file0 + + + + + + RegexpComparator + touchz: `file0': No such file or directory + + + + touchz: touching file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -du file0 @@ -6051,6 +6111,7 @@ touchz: touching many files + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 file1 file2 -fs NAMENODE -du file* @@ -6070,6 +6131,7 @@ touchz: touching already existing file + -fs NAMENODE -mkdir -p dir -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -touchz data15bytes @@ -6087,6 +6149,7 @@ touchz: Test for hdfs:// path - touching file + -fs NAMENODE -mkdir -p hdfs:///user/ -fs NAMENODE -touchz hdfs:///user/file0 -fs NAMENODE -du hdfs:///user/file0 @@ -6340,6 +6403,7 @@ stat: statistics about file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data60bytes data60bytes -fs NAMENODE -stat "%n-%b" data60bytes @@ -6430,6 +6494,7 @@ stat: statistics about files (relative path) using globbing + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes data30bytes @@ -6690,6 +6755,7 @@ tail: contents of file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -tail data15bytes @@ -6968,6 +7034,7 @@ count: file using relative path + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -count file1 @@ -7145,6 +7212,7 @@ count: relative path to multiple files without globbing + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 -fs NAMENODE -touchz file3 @@ -7322,6 +7390,7 @@ count: file using relative path with -q option + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -count -q file1 @@ -7503,6 +7572,7 @@ count: relative path to multiple files without globbing with -q option + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 -fs NAMENODE -touchz file3 @@ -8462,6 +8532,7 @@ chmod: change permission(octal mode) of file in relative path + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -chmod 666 file1 -fs NAMENODE -ls file1 @@ -15988,6 +16059,7 @@ moveFromLocal: moving non existent file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -moveFromLocal wrongdata file @@ -16007,6 +16079,7 @@ -fs NAMENODE -moveFromLocal CLITEST_DATA/data15bytes /data15bytes -fs NAMENODE -moveFromLocal CLITEST_DATA/data30bytes /data30bytes + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -moveFromLocal /data15bytes /data30bytes file0 From 7226dead1e66e501077fe1a0ec04369f03ecd2d2 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 30 Jul 2012 20:58:44 +0000 Subject: [PATCH 36/39] HADOOP-8635. Cannot cancel paths registered deleteOnExit (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367296 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../java/org/apache/hadoop/fs/FileSystem.java | 10 +++ .../hadoop/fs/TestFileSystemCaching.java | 88 +++++++++++++++---- .../hadoop/fs/TestFilterFileSystem.java | 3 + 4 files changed, 87 insertions(+), 16 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d1faef370a6..03e41de3da7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -865,6 +865,8 @@ Release 0.23.3 - UNRELEASED HADOOP-8550. hadoop fs -touchz automatically created parent directories (John George via bobby) + HADOOP-8635. Cannot cancel paths registered deleteOnExit (daryn via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 42b30c4999b..2fa4bd0c038 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -1214,6 +1214,16 @@ public abstract class FileSystem extends Configured implements Closeable { } return true; } + + /** + * Cancel the deletion of the path when the FileSystem is closed + * @param f the path to cancel deletion + */ + public boolean cancelDeleteOnExit(Path f) { + synchronized (deleteOnExit) { + return deleteOnExit.remove(f); + } + } /** * Delete all files that were marked as delete-on-exit. This recursively diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java index e8debf8dfaf..7e5f99f5fb3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java @@ -269,26 +269,82 @@ public class TestFileSystemCaching { } @Test - public void testDeleteOnExitChecksExists() throws Exception { + public void testDelete() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + fs.delete(path, false); + verify(mockFs).delete(eq(path), eq(false)); + reset(mockFs); + fs.delete(path, true); + verify(mockFs).delete(eq(path), eq(true)); + } + + @Test + public void testDeleteOnExit() throws IOException { FileSystem mockFs = mock(FileSystem.class); FileSystem fs = new FilterFileSystem(mockFs); - Path p = new Path("/a"); - - // path has to exist for deleteOnExit to register it - when(mockFs.getFileStatus(p)).thenReturn(new FileStatus()); - fs.deleteOnExit(p); - verify(mockFs).getFileStatus(eq(p)); - fs.close(); - verify(mockFs).delete(eq(p), anyBoolean()); + Path path = new Path("/a"); + + // delete on close if path does exist + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + assertTrue(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); reset(mockFs); - - // make sure it doesn't try to delete a file that doesn't exist - when(mockFs.getFileStatus(p)).thenReturn(new FileStatus()); - fs.deleteOnExit(p); - verify(mockFs).getFileStatus(eq(p)); + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + fs.close(); + verify(mockFs).getFileStatus(eq(path)); + verify(mockFs).delete(eq(path), eq(true)); + } + + @Test + public void testDeleteOnExitFNF() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + // don't delete on close if path doesn't exist + assertFalse(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); reset(mockFs); fs.close(); - verify(mockFs).getFileStatus(eq(p)); + verify(mockFs, never()).getFileStatus(eq(path)); verify(mockFs, never()).delete(any(Path.class), anyBoolean()); } -} \ No newline at end of file + + + @Test + public void testDeleteOnExitRemoved() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + // don't delete on close if path existed, but later removed + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + assertTrue(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); + reset(mockFs); + fs.close(); + verify(mockFs).getFileStatus(eq(path)); + verify(mockFs, never()).delete(any(Path.class), anyBoolean()); + } + + @Test + public void testCancelDeleteOnExit() throws IOException { + FileSystem mockFs = mock(FileSystem.class); + FileSystem fs = new FilterFileSystem(mockFs); + Path path = new Path("/a"); + + // don't delete on close if path existed, but later cancelled + when(mockFs.getFileStatus(eq(path))).thenReturn(new FileStatus()); + assertTrue(fs.deleteOnExit(path)); + verify(mockFs).getFileStatus(eq(path)); + assertTrue(fs.cancelDeleteOnExit(path)); + assertFalse(fs.cancelDeleteOnExit(path)); // false because not registered + reset(mockFs); + fs.close(); + verify(mockFs, never()).getFileStatus(any(Path.class)); + verify(mockFs, never()).delete(any(Path.class), anyBoolean()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index a374c08a186..727986dcd12 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -182,6 +182,9 @@ public class TestFilterFileSystem { public boolean deleteOnExit(Path f) throws IOException { return false; } + public boolean cancelDeleteOnExit(Path f) throws IOException { + return false; + } public String getScheme() { return "dontcheck"; } From 8a615793108ee560fb11efa5446ef0afe0b8413b Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Mon, 30 Jul 2012 22:38:43 +0000 Subject: [PATCH 37/39] Fix typo in CHANGES.txt: wrote HADOOP-8642 instead of HADOOP-8624. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367347 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 03e41de3da7..cda7817987d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -88,7 +88,7 @@ Trunk (unreleased changes) HADOOP-8523. test-patch.sh doesn't validate patches before building (Jack Dintruff via jeagles) - HADOOP-8642. ProtobufRpcEngine should log all RPCs if TRACE logging is + HADOOP-8624. ProtobufRpcEngine should log all RPCs if TRACE logging is enabled (todd) BUG FIXES From 56c7662b18b36941e6a6a2397624380a04f3bc48 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 30 Jul 2012 22:44:48 +0000 Subject: [PATCH 38/39] svn ignoring hadoop-yarn-applications-unmanaged-am-launcher target directory (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367349 13f79535-47bb-0310-9956-ffa450edef68 From e217a05255497fcf243ad7b379553db7fd0a2493 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 30 Jul 2012 22:56:15 +0000 Subject: [PATCH 39/39] MAPREDUCE-4342. Distributed Cache gives inconsistent result if cache files get deleted from tasktracker. (mayank_bansal via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367352 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../localizer/LocalResourcesTrackerImpl.java | 26 ++++++ .../TestLocalResourcesTrackerImpl.java | 83 +++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 620ccb84a33..8ca1299e6b5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -156,6 +156,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4465. Update description of yarn.nodemanager.address property. (bowang via tucu) + MAPREDUCE-4342. Distributed Cache gives inconsistent result if cache files + get deleted from tasktracker. (mayank_bansal via tucu) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java index 7127db97c08..01ec38397b6 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; +import java.io.File; import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEventType; /** * A collection of {@link LocalizedResource}s all of same @@ -67,6 +69,12 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker { switch (event.getType()) { case REQUEST: case LOCALIZED: + if (rsrc != null && (!isResourcePresent(rsrc))) { + LOG.info("Resource " + rsrc.getLocalPath() + + " is missing, localizing it again"); + localrsrc.remove(req); + rsrc = null; + } if (null == rsrc) { rsrc = new LocalizedResource(req, dispatcher); localrsrc.put(req, rsrc); @@ -82,6 +90,24 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker { rsrc.handle(event); } + /** + * This module checks if the resource which was localized is already present + * or not + * + * @param rsrc + * @return true/false based on resource is present or not + */ + public boolean isResourcePresent(LocalizedResource rsrc) { + boolean ret = true; + if (rsrc.getState() == ResourceState.LOCALIZED) { + File file = new File(rsrc.getLocalPath().toUri().getRawPath().toString()); + if (!file.exists()) { + ret = false; + } + } + return ret; + } + @Override public boolean contains(LocalResourceRequest resource) { return localrsrc.containsKey(resource); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java index 7f0e3536975..3ee623cecaf 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java @@ -5,6 +5,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import java.io.File; +import java.io.IOException; import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -30,6 +32,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; +import org.mortbay.log.Log; public class TestLocalResourcesTrackerImpl { @@ -131,6 +134,86 @@ public class TestLocalResourcesTrackerImpl { } } + @Test + @SuppressWarnings("unchecked") + public void testConsistency() { + String user = "testuser"; + DrainDispatcher dispatcher = null; + try { + dispatcher = createDispatcher(new Configuration()); + EventHandler localizerEventHandler = mock(EventHandler.class); + EventHandler containerEventHandler = mock(EventHandler.class); + dispatcher.register(LocalizerEventType.class, localizerEventHandler); + dispatcher.register(ContainerEventType.class, containerEventHandler); + + ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); + LocalizerContext lc1 = new LocalizerContext(user, cId1, null); + LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, + LocalResourceVisibility.PUBLIC); + LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); + ConcurrentMap localrsrc = new ConcurrentHashMap(); + localrsrc.put(req1, lr1); + LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, + dispatcher, localrsrc); + + ResourceEvent req11Event = new ResourceRequestEvent(req1, + LocalResourceVisibility.PUBLIC, lc1); + + ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); + + // Localize R1 for C1 + tracker.handle(req11Event); + + dispatcher.await(); + + // Verify refCount for R1 is 1 + Assert.assertEquals(1, lr1.getRefCount()); + + dispatcher.await(); + verifyTrackedResourceCount(tracker, 1); + + // Localize resource1 + ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path( + "file:///tmp/r1"), 1); + lr1.handle(rle); + Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); + Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1"))); + LocalizedResource rsrcbefore = tracker.iterator().next(); + File resFile = new File(lr1.getLocalPath().toUri().getRawPath() + .toString()); + Assert.assertTrue(resFile.exists()); + Assert.assertTrue(resFile.delete()); + + // Localize R1 for C1 + tracker.handle(req11Event); + + dispatcher.await(); + lr1.handle(rle); + Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); + LocalizedResource rsrcafter = tracker.iterator().next(); + if (rsrcbefore == rsrcafter) { + Assert.fail("Localized resource should not be equal"); + } + // Release resource1 + tracker.handle(rel11Event); + } finally { + if (dispatcher != null) { + dispatcher.stop(); + } + } + } + + private boolean createdummylocalizefile(Path path) { + boolean ret = false; + File file = new File(path.toUri().getRawPath().toString()); + try { + ret = file.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + return ret; + } + private void verifyTrackedResourceCount(LocalResourcesTracker tracker, int expected) { int count = 0;