diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index 72222be04a8..4f4c937b440 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -40,6 +40,8 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Time; +import com.google.common.annotations.VisibleForTesting; + /** Provides a trash feature. Files are moved to a user's trash * directory, a subdirectory of their home directory named ".Trash". Files are * initially moved to a current sub-directory of the trash directory. @@ -215,7 +217,7 @@ public class TrashPolicyDefault extends TrashPolicy { return new Emptier(getConf(), emptierInterval); } - private class Emptier implements Runnable { + protected class Emptier implements Runnable { private Configuration conf; private long emptierInterval; @@ -223,7 +225,7 @@ public class TrashPolicyDefault extends TrashPolicy { Emptier(Configuration conf, long emptierInterval) throws IOException { this.conf = conf; this.emptierInterval = emptierInterval; - if (emptierInterval > deletionInterval || emptierInterval == 0) { + if (emptierInterval > deletionInterval || emptierInterval <= 0) { LOG.info("The configured checkpoint interval is " + (emptierInterval / MSECS_PER_MINUTE) + " minutes." + " Using an interval of " + @@ -287,6 +289,11 @@ public class TrashPolicyDefault extends TrashPolicy { private long floor(long time, long interval) { return (time / interval) * interval; } + + @VisibleForTesting + protected long getEmptierInterval() { + return this.emptierInterval/MSECS_PER_MINUTE; + } } private void createCheckpoint(Path trashRoot, Date date) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java index 338aff6e8d4..7a5b25e583a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java @@ -29,13 +29,19 @@ import java.net.URI; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.HashSet; +import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.TrashPolicyDefault.Emptier; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; +import org.junit.Before; +import org.junit.Test; /** * This class tests commands from Trash. @@ -45,6 +51,13 @@ public class TestTrash extends TestCase { private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath( "testTrash")); + @Before + public void setUp() throws IOException { + // ensure each test initiates a FileSystem instance, + // avoid getting an old instance from cache. + FileSystem.closeAll(); + } + protected static Path mkdir(FileSystem fs, Path p) throws IOException { assertTrue(fs.mkdirs(p)); assertTrue(fs.exists(p)); @@ -516,6 +529,81 @@ public class TestTrash extends TestCase { assertTrue(trash.getTrashPolicy().getClass().equals(TestTrashPolicy.class)); } + @Test + public void testCheckpointInterval() throws IOException { + // Verify if fs.trash.checkpoint.interval is set to positive number + // but bigger than fs.trash.interval, + // the value should be reset to fs.trash.interval + verifyDefaultPolicyIntervalValues(10, 12, 10); + + // Verify if fs.trash.checkpoint.interval is set to positive number + // and smaller than fs.trash.interval, the value should be respected + verifyDefaultPolicyIntervalValues(10, 5, 5); + + // Verify if fs.trash.checkpoint.interval sets to 0 + // the value should be reset to fs.trash.interval + verifyDefaultPolicyIntervalValues(10, 0, 10); + + // Verify if fs.trash.checkpoint.interval sets to a negative number + // the value should be reset to fs.trash.interval + verifyDefaultPolicyIntervalValues(10, -1, 10); + } + + @Test + public void testMoveEmptyDirToTrash() throws Exception { + Configuration conf = new Configuration(); + conf.setClass(FS_FILE_IMPL_KEY, + RawLocalFileSystem.class, + FileSystem.class); + conf.setLong(FS_TRASH_INTERVAL_KEY, 1); // 1 min + FileSystem fs = FileSystem.get(conf); + verifyMoveEmptyDirToTrash(fs, conf); + } + + /** + * Simulate the carrier process of the trash emptier restarts, + * verify it honors the fs.trash.interval before and after restart. + * @throws Exception + */ + @Test + public void testTrashRestarts() throws Exception { + Configuration conf = new Configuration(); + conf.setClass("fs.trash.classname", + AuditableTrashPolicy.class, + TrashPolicy.class); + conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class); + conf.set(FS_TRASH_INTERVAL_KEY, "50"); // in milliseconds for test + Trash trash = new Trash(conf); + // create 5 checkpoints + for(int i=0; i<5; i++) { + trash.checkpoint(); + } + + // Run the trash emptier for 120ms, it should run + // 2 times deletion as the interval is 50ms. + // Verify the checkpoints number when shutting down the emptier. + verifyAuditableTrashEmptier(trash, 120, 3); + + // reconfigure the interval to 100 ms + conf.set(FS_TRASH_INTERVAL_KEY, "100"); + Trash trashNew = new Trash(conf); + + // Run the trash emptier for 120ms, it should run + // 1 time deletion. + verifyAuditableTrashEmptier(trashNew, 120, 2); + } + + @Test + public void testTrashPermission() throws IOException { + Configuration conf = new Configuration(); + conf.setClass("fs.trash.classname", + TrashPolicyDefault.class, + TrashPolicy.class); + conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class); + conf.set(FS_TRASH_INTERVAL_KEY, "0.2"); + verifyTrashPermission(FileSystem.getLocal(conf), conf); + } + public void testTrashEmptier() throws Exception { Configuration conf = new Configuration(); // Trash with 12 second deletes and 6 seconds checkpoints @@ -679,12 +767,143 @@ public class TestTrash extends TestCase { long factoredTime = first*factor; assertTrue(iterTime 0) { + numOfCheckpoint.decrementAndGet(); + System.out.println(String + .format("Delete a checkpoint, current number of checkpoints %d", + numOfCheckpoint.get())); + } + } + + private static int get() { + return numOfCheckpoint.get(); + } + } }