From c6e4906ffc59e39330e6b27c8e977a199ba80221 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Wed, 14 Aug 2019 07:24:22 -0700 Subject: [PATCH] HDFS-14595. HDFS-11848 breaks API compatibility. Contributed by Siyao Meng. Signed-off-by: Wei-Chiu Chuang Reviewed-by: Ayush Saxena (cherry picked from commit 3c0382f1b933b7acfe55081f5bad46f9fe05a14b) --- .../hadoop/hdfs/DistributedFileSystem.java | 6 +++ .../apache/hadoop/hdfs/client/HdfsAdmin.java | 6 +++ .../hdfs/TestDistributedFileSystem.java | 17 +++++++++ .../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 38 +++++++++++++++---- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 5e08ce41e73..e506552c855 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -3332,6 +3332,12 @@ public class DistributedFileSystem extends FileSystem return dfs.listOpenFiles(); } + @Deprecated + public RemoteIterator listOpenFiles( + EnumSet openFilesTypes) throws IOException { + return dfs.listOpenFiles(openFilesTypes); + } + public RemoteIterator listOpenFiles( EnumSet openFilesTypes, String path) throws IOException { return dfs.listOpenFiles(openFilesTypes, path); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 575cd24789e..3b3d563cd40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -626,6 +626,12 @@ public class HdfsAdmin { return dfs.listOpenFiles(); } + @Deprecated + public RemoteIterator listOpenFiles( + EnumSet openFilesTypes) throws IOException { + return dfs.listOpenFiles(openFilesTypes); + } + public RemoteIterator listOpenFiles( EnumSet openFilesTypes, String path) throws IOException { return dfs.listOpenFiles(openFilesTypes, path); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 2a4457f7750..394ec388c49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -86,6 +86,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -184,8 +186,10 @@ public class TestDistributedFileSystem { * Tests DFSClient.close throws no ConcurrentModificationException if * multiple files are open. * Also tests that any cached sockets are closed. (HDFS-3359) + * Also tests deprecated listOpenFiles(EnumSet<>). (HDFS-14595) */ @Test + @SuppressWarnings("deprecation") // call to listOpenFiles(EnumSet<>) public void testDFSClose() throws Exception { Configuration conf = getTestConfiguration(); MiniDFSCluster cluster = null; @@ -197,6 +201,19 @@ public class TestDistributedFileSystem { fileSys.create(new Path("/test/dfsclose/file-0")); fileSys.create(new Path("/test/dfsclose/file-1")); + // Test listOpenFiles(EnumSet<>) + List types = new ArrayList<>(); + types.add(OpenFilesIterator.OpenFilesType.ALL_OPEN_FILES); + RemoteIterator listOpenFiles = + fileSys.listOpenFiles(EnumSet.copyOf(types)); + assertTrue("Two files should be open", listOpenFiles.hasNext()); + int countOpenFiles = 0; + while (listOpenFiles.hasNext()) { + listOpenFiles.next(); + ++countOpenFiles; + } + assertEquals("Mismatch of open files count", 2, countOpenFiles); + // create another file, close it, and read it, so // the client gets a socket in its SocketCache Path p = new Path("/non-empty-file"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java index cc32a3cf44d..9edd2979bde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java @@ -233,6 +233,9 @@ public class TestHdfsAdmin { closedFileSet.add(filePath); } verifyOpenFiles(closedFileSet, openFileMap); + // Verify again with the old listOpenFiles(EnumSet<>) API + // Just to verify old API's validity + verifyOpenFilesOld(closedFileSet, openFileMap); openFileMap.putAll( DFSTestUtil.createOpenFiles(fs, "open-file-1", numOpenFiles)); @@ -252,13 +255,10 @@ public class TestHdfsAdmin { } } - private void verifyOpenFiles(HashSet closedFiles, - HashMap openFileMap) throws IOException { - HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); - HashSet openFiles = new HashSet<>(openFileMap.keySet()); - RemoteIterator openFilesRemoteItr = - hdfsAdmin.listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES), - OpenFilesIterator.FILTER_PATH_DEFAULT); + private void verifyOpenFilesHelper( + RemoteIterator openFilesRemoteItr, + HashSet closedFiles, + HashSet openFiles) throws IOException { while (openFilesRemoteItr.hasNext()) { String filePath = openFilesRemoteItr.next().getFilePath(); assertFalse(filePath + " should not be listed under open files!", @@ -266,6 +266,30 @@ public class TestHdfsAdmin { assertTrue(filePath + " is not listed under open files!", openFiles.remove(new Path(filePath))); } + } + + private void verifyOpenFiles(HashSet closedFiles, + HashMap openFileMap) throws IOException { + HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); + HashSet openFiles = new HashSet<>(openFileMap.keySet()); + RemoteIterator openFilesRemoteItr = + hdfsAdmin.listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); + verifyOpenFilesHelper(openFilesRemoteItr, closedFiles, openFiles); + assertTrue("Not all open files are listed!", openFiles.isEmpty()); + } + + /** + * Using deprecated HdfsAdmin#listOpenFiles(EnumSet<>) to verify open files. + */ + @SuppressWarnings("deprecation") // call to listOpenFiles(EnumSet<>) + private void verifyOpenFilesOld(HashSet closedFiles, + HashMap openFileMap) throws IOException { + HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); + HashSet openFiles = new HashSet<>(openFileMap.keySet()); + RemoteIterator openFilesRemoteItr = + hdfsAdmin.listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + verifyOpenFilesHelper(openFilesRemoteItr, closedFiles, openFiles); assertTrue("Not all open files are listed!", openFiles.isEmpty()); } }