diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java index 5c879ecf414..67e86904563 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java @@ -740,6 +740,7 @@ public class CryptoInputStream extends FilterInputStream implements case StreamCapabilities.READAHEAD: case StreamCapabilities.DROPBEHIND: case StreamCapabilities.UNBUFFER: + case StreamCapabilities.READBYTEBUFFER: return true; default: return false; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java index 3549cdc4fa3..c52d30762f0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java @@ -59,6 +59,12 @@ public interface StreamCapabilities { */ String UNBUFFER = "in:unbuffer"; + /** + * Stream read(ByteBuffer) capability implemented by + * {@link ByteBufferReadable#read(java.nio.ByteBuffer)}. + */ + String READBYTEBUFFER = "in:readbytebuffer"; + /** * Capabilities that a stream can support and be queried for. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index f47b88cb5a2..a3e2ad5afca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -1779,6 +1779,7 @@ public class DFSInputStream extends FSInputStream case StreamCapabilities.READAHEAD: case StreamCapabilities.DROPBEHIND: case StreamCapabilities.UNBUFFER: + case StreamCapabilities.READBYTEBUFFER: return true; default: return false; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c index 0cced979ad9..41caffd290c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c @@ -1013,7 +1013,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, return f{is|os}; */ int accmode = flags & O_ACCMODE; - jstring jStrBufferSize = NULL, jStrReplication = NULL; + jstring jStrBufferSize = NULL, jStrReplication = NULL, jCapabilityString = NULL; jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; jobject jFS = (jobject)fs; jthrowable jthr; @@ -1171,16 +1171,22 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, file->flags = 0; if ((flags & O_WRONLY) == 0) { - // Try a test read to see if we can do direct reads - char buf; - if (readDirect(fs, file, &buf, 0) == 0) { - // Success - 0-byte read should return 0 + // Check the StreamCapabilities of jFile to see if we can do direct reads + jthr = newJavaStr(env, "in:readbytebuffer", &jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): newJavaStr", path); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFile, HADOOP_ISTRM, + "hasCapability", "(Ljava/lang/String;)Z", jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FSDataInputStream#hasCapability", path); + goto done; + } + if (jVal.z) { file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; - } else if (errno != ENOTSUP) { - // Unexpected error. Clear it, don't set the direct flag. - fprintf(stderr, - "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " - "for direct read compatibility\n", path, errno); } } ret = 0; @@ -1190,7 +1196,8 @@ done: destroyLocalReference(env, jStrReplication); destroyLocalReference(env, jConfiguration); destroyLocalReference(env, jPath); - destroyLocalReference(env, jFile); + destroyLocalReference(env, jFile); + destroyLocalReference(env, jCapabilityString); if (ret) { if (file) { if (file->file) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc index 19d95b47e69..79771f0d7c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc @@ -503,7 +503,10 @@ TEST_F(HdfsExtTest, TestReadStats) { hdfsFileFreeReadStatistics(stats); EXPECT_EQ(0, hdfsCloseFile(fs, file)); - EXPECT_EQ(0, errno); + // Since libhdfs is not guaranteed to set errno to 0 on successful + // operations, we disable this check for now, see HDFS-14325 for a + // long term solution to this problem + // EXPECT_EQ(0, errno); } //Testing working directory