diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java index 53eefa0425b..dee17a9192e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java @@ -35,7 +35,7 @@ import java.util.List; @InterfaceStability.Unstable public abstract class Event { public static enum EventType { - CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK + CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK, TRUNCATE } private EventType eventType; @@ -542,4 +542,39 @@ public abstract class Event { return timestamp; } } + + /** + * Sent when a file is truncated. + */ + public static class TruncateEvent extends Event { + private String path; + private long fileSize; + private long timestamp; + + + public TruncateEvent(String path, long fileSize, long timestamp) { + super(EventType.TRUNCATE); + this.path = path; + this.fileSize = fileSize; + this.timestamp = timestamp; + } + + public String getPath() { + return path; + } + + /** + * The size of the truncated file in bytes. + */ + public long getFileSize() { + return fileSize; + } + + /** + * The time when this event occurred, in milliseconds since the epoch. + */ + public long getTimestamp() { + return timestamp; + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto index 5b78fe62de6..53399029582 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto @@ -41,6 +41,7 @@ enum EventType { EVENT_RENAME = 0x3; EVENT_METADATA = 0x4; EVENT_UNLINK = 0x5; + EVENT_TRUNCATE = 0x6; } message EventProto { @@ -87,6 +88,12 @@ message CloseEventProto { required int64 timestamp = 3; } +message TruncateEventProto { + required string path = 1; + required int64 fileSize = 2; + required int64 timestamp = 3; +} + message AppendEventProto { required string path = 1; optional bool newBlock = 2 [default = false]; diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 403ed062482..548e4737c86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -380,6 +380,9 @@ Release 2.8.0 - UNRELEASED HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move progress. (Surendra Singh Lilhore via szetszwo) + HDFS-8742. Inotify: Support event for OP_TRUNCATE. + (Surendra Singh Lilhore via aajisaka) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index e46a3a56e0a..a2522622879 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -2686,6 +2686,12 @@ public class PBHelper { .timestamp(unlink.getTimestamp()) .build()); break; + case EVENT_TRUNCATE: + InotifyProtos.TruncateEventProto truncate = + InotifyProtos.TruncateEventProto.parseFrom(p.getContents()); + events.add(new Event.TruncateEvent(truncate.getPath(), + truncate.getFileSize(), truncate.getTimestamp())); + break; default: throw new RuntimeException("Unexpected inotify event type: " + p.getType()); @@ -2792,6 +2798,17 @@ public class PBHelper { .setTimestamp(ue.getTimestamp()).build().toByteString() ).build()); break; + case TRUNCATE: + Event.TruncateEvent te = (Event.TruncateEvent) e; + events.add(InotifyProtos.EventProto.newBuilder() + .setType(InotifyProtos.EventType.EVENT_TRUNCATE) + .setContents( + InotifyProtos.TruncateEventProto.newBuilder() + .setPath(te.getPath()) + .setFileSize(te.getFileSize()) + .setTimestamp(te.getTimestamp()).build().toByteString() + ).build()); + break; default: throw new RuntimeException("Unexpected inotify event: " + e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java index 5345b46de5c..09181074643 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java @@ -176,6 +176,10 @@ public class InotifyFSEditLogOpTranslator { .metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS) .path(saOp.src) .acls(saOp.aclEntries).build() }); + case OP_TRUNCATE: + FSEditLogOp.TruncateOp tOp = (FSEditLogOp.TruncateOp) op; + return new EventBatch(op.txid, new Event[] { + new Event.TruncateEvent(tOp.src, tOp.newLength, tOp.timestamp) }); default: return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java index ba33bd3ec48..385d6534783 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java @@ -102,6 +102,8 @@ public class TestDFSInotifyEventInputStream { DFSTestUtil.createFile(fs, new Path("/file"), BLOCK_SIZE, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path("/file3"), BLOCK_SIZE, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path("/file5"), BLOCK_SIZE, (short) 1, 0L); + DFSTestUtil.createFile(fs, new Path("/truncate_file"), + BLOCK_SIZE * 2, (short) 1, 0L); DFSInotifyEventInputStream eis = client.getInotifyEventStream(); client.rename("/file", "/file4", null); // RenameOp -> RenameEvent client.rename("/file4", "/file2"); // RenameOldOp -> RenameEvent @@ -136,7 +138,8 @@ public class TestDFSInotifyEventInputStream { "user::rwx,user:foo:rw-,group::r--,other::---", true)); client.removeAcl("/file5"); // SetAclOp -> MetadataUpdateEvent client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent - + //TruncateOp -> TruncateEvent + client.truncate("/truncate_file", BLOCK_SIZE); EventBatch batch = null; // RenameOp @@ -354,6 +357,18 @@ public class TestDFSInotifyEventInputStream { Assert.assertTrue(re3.getSrcPath().equals("/file5")); Assert.assertTrue(re.getTimestamp() > 0); + // TruncateOp + batch = waitForNextEvents(eis); + Assert.assertEquals(1, batch.getEvents().length); + txid = checkTxid(batch, txid); + Assert + .assertTrue(batch.getEvents()[0].getEventType() == + Event.EventType.TRUNCATE); + Event.TruncateEvent et = ((Event.TruncateEvent) batch.getEvents()[0]); + Assert.assertTrue(et.getPath().equals("/truncate_file")); + Assert.assertTrue(et.getFileSize() == BLOCK_SIZE); + Assert.assertTrue(et.getTimestamp() > 0); + // Returns null when there are no further events Assert.assertTrue(eis.poll() == null);