From e226dbbc8ae99eb3a48e2e27887d284dc193e9bd Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Wed, 17 Feb 2016 13:30:50 -0800 Subject: [PATCH] HDFS-9799. Reimplement getCurrentTrashDir to remove incompatibility. (zhz) Change-Id: I7834bcebffed38cb384db5395ddb8b6dd9e79a0b --- .../java/org/apache/hadoop/fs/FileSystem.java | 41 ++++++------ .../apache/hadoop/fs/FilterFileSystem.java | 5 +- .../org/apache/hadoop/fs/TrashPolicy.java | 2 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 9 +-- .../hadoop/hdfs/DistributedFileSystem.java | 63 +++++++++++-------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + 6 files changed, 64 insertions(+), 58 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index d9cb9e4ab72..a97c94139a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2675,9 +2675,8 @@ public abstract class FileSystem extends Configured implements Closeable { * * @param path the trash root of the path to be determined. * @return the default implementation returns "/user/$USER/.Trash". - * @throws IOException */ - public Path getTrashRoot(Path path) throws IOException { + public Path getTrashRoot(Path path) { return this.makeQualified(new Path(getHomeDirectory().toUri().getPath(), TRASH_PREFIX)); } @@ -2689,29 +2688,31 @@ public abstract class FileSystem extends Configured implements Closeable { * @return all the trash root directories. * Default FileSystem returns .Trash under users' home directories if * /user/$USER/.Trash exists. - * @throws IOException */ - public Collection getTrashRoots(boolean allUsers) - throws IOException { + public Collection getTrashRoots(boolean allUsers) { Path userHome = new Path(getHomeDirectory().toUri().getPath()); - List ret = new ArrayList(); - if (!allUsers) { - Path userTrash = new Path(userHome, TRASH_PREFIX); - if (exists(userTrash)) { - ret.add(getFileStatus(userTrash)); - } - } else { - Path homeParent = userHome.getParent(); - if (exists(homeParent)) { - FileStatus[] candidates = listStatus(homeParent); - for (FileStatus candidate : candidates) { - Path userTrash = new Path(candidate.getPath(), TRASH_PREFIX); - if (exists(userTrash)) { - candidate.setPath(userTrash); - ret.add(candidate); + List ret = new ArrayList<>(); + try { + if (!allUsers) { + Path userTrash = new Path(userHome, TRASH_PREFIX); + if (exists(userTrash)) { + ret.add(getFileStatus(userTrash)); + } + } else { + Path homeParent = userHome.getParent(); + if (exists(homeParent)) { + FileStatus[] candidates = listStatus(homeParent); + for (FileStatus candidate : candidates) { + Path userTrash = new Path(candidate.getPath(), TRASH_PREFIX); + if (exists(userTrash)) { + candidate.setPath(userTrash); + ret.add(candidate); + } } } } + } catch (IOException e) { + LOG.warn("Cannot get all trash roots", e); } return ret; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 53678e018b3..4ee75144727 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -645,13 +645,12 @@ public class FilterFileSystem extends FileSystem { } @Override - public Path getTrashRoot(Path path) throws IOException { + public Path getTrashRoot(Path path) { return fs.getTrashRoot(path); } @Override - public Collection getTrashRoots(boolean allUsers) - throws IOException { + public Collection getTrashRoots(boolean allUsers) { return fs.getTrashRoots(allUsers); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java index 1d901c13717..92a4d1f2822 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java @@ -89,7 +89,7 @@ public abstract class TrashPolicy extends Configured { * It returns the trash location correctly for the path specified no matter * the path is in encryption zone or not. */ - public abstract Path getCurrentTrashDir() throws IOException; + public abstract Path getCurrentTrashDir(); /** * Get the current trash directory for path specified based on the Trash diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index b5d245b86eb..012ce32df1f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -196,7 +196,7 @@ public class TrashPolicyDefault extends TrashPolicy { } @Override - public Path getCurrentTrashDir() throws IOException { + public Path getCurrentTrashDir() { return new Path(fs.getTrashRoot(null), CURRENT); } @@ -250,12 +250,7 @@ public class TrashPolicyDefault extends TrashPolicy { now = Time.now(); if (now >= end) { Collection trashRoots; - try { - trashRoots = fs.getTrashRoots(true); // list all home dirs - } catch (IOException e) { - LOG.warn("Trash can't list all trash roots: "+e+" Sleeping."); - continue; - } + trashRoots = fs.getTrashRoots(true); // list all trash dirs for (FileStatus trashRoot : trashRoots) { // dump each trash if (!trashRoot.isDirectory()) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index b7d4de72199..e2449c7d6b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2333,27 +2333,31 @@ public class DistributedFileSystem extends FileSystem { /** * Get the root directory of Trash for a path in HDFS. * 1. File in encryption zone returns /ez1/.Trash/username - * 2. File not in encryption zone returns /users/username/.Trash + * 2. File not in encryption zone, or encountered exception when checking + * the encryption zone of the path, returns /users/username/.Trash * Caller appends either Current or checkpoint timestamp for trash destination * @param path the trash root of the path to be determined. * @return trash root - * @throws IOException */ @Override - public Path getTrashRoot(Path path) throws IOException { + public Path getTrashRoot(Path path) { if ((path == null) || path.isRoot() || !dfs.isHDFSEncryptionEnabled()) { return super.getTrashRoot(path); } String parentSrc = path.getParent().toUri().getPath(); - EncryptionZone ez = dfs.getEZForPath(parentSrc); - if ((ez != null)) { - return this.makeQualified( - new Path(ez.getPath() + "/" + FileSystem.TRASH_PREFIX + - dfs.ugi.getShortUserName())); - } else { - return super.getTrashRoot(path); + try { + EncryptionZone ez = dfs.getEZForPath(parentSrc); + if ((ez != null)) { + return this.makeQualified( + new Path(ez.getPath() + "/" + FileSystem.TRASH_PREFIX + + dfs.ugi.getShortUserName())); + } + } catch (IOException e) { + DFSClient.LOG.warn("Exception in checking the encryption zone for the " + + "path " + parentSrc + ". " + e.getMessage()); } + return super.getTrashRoot(path); } /** @@ -2361,32 +2365,37 @@ public class DistributedFileSystem extends FileSystem { * 1. File deleted from non-encryption zone /user/username/.Trash * 2. File deleted from encryption zones * e.g., ez1 rooted at /ez1 has its trash root at /ez1/.Trash/$USER - * @allUsers return trashRoots of all users if true, used by emptier + * @param allUsers return trashRoots of all users if true, used by emptier * @return trash roots of HDFS - * @throws IOException */ @Override - public Collection getTrashRoots(boolean allUsers) throws IOException { - List ret = new ArrayList(); + public Collection getTrashRoots(boolean allUsers) { + List ret = new ArrayList<>(); // Get normal trash roots ret.addAll(super.getTrashRoots(allUsers)); - // Get EZ Trash roots - final RemoteIterator it = dfs.listEncryptionZones(); - while (it.hasNext()) { - Path ezTrashRoot = new Path(it.next().getPath(), FileSystem.TRASH_PREFIX); - if (allUsers) { - for (FileStatus candidate : listStatus(ezTrashRoot)) { - if (exists(candidate.getPath())) { - ret.add(candidate); + try { + // Get EZ Trash roots + final RemoteIterator it = dfs.listEncryptionZones(); + while (it.hasNext()) { + Path ezTrashRoot = new Path(it.next().getPath(), + FileSystem.TRASH_PREFIX); + if (allUsers) { + for (FileStatus candidate : listStatus(ezTrashRoot)) { + if (exists(candidate.getPath())) { + ret.add(candidate); + } + } + } else { + Path userTrash = new Path(ezTrashRoot, System.getProperty( + "user.name")); + if (exists(userTrash)) { + ret.add(getFileStatus(userTrash)); } } - } else { - Path userTrash = new Path(ezTrashRoot, System.getProperty("user.name")); - if (exists(userTrash)) { - ret.add(getFileStatus(userTrash)); - } } + } catch (IOException e){ + DFSClient.LOG.warn("Cannot get all encrypted trash roots", e); } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a6b33638741..e929f1beb93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1856,6 +1856,8 @@ Release 2.8.0 - UNRELEASED HDFS-9815. Move o.a.h.fs.Hdfs to hadoop-hdfs-client. (Vinayakumar B via wheat9) + HDFS-9799. Reimplement getCurrentTrashDir to remove incompatibility. (zhz) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES