HDFS-8742. Inotify: Support event for OP_TRUNCATE. Contributed by Surendra Singh Lilhore.
(cherry picked from commit 979c9ca2ca
)
This commit is contained in:
parent
b0a2dc9c84
commit
bad5d288ac
|
@ -35,7 +35,7 @@ import java.util.List;
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public abstract class Event {
|
public abstract class Event {
|
||||||
public static enum EventType {
|
public static enum EventType {
|
||||||
CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK
|
CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK, TRUNCATE
|
||||||
}
|
}
|
||||||
|
|
||||||
private EventType eventType;
|
private EventType eventType;
|
||||||
|
@ -542,4 +542,39 @@ public abstract class Event {
|
||||||
return timestamp;
|
return timestamp;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sent when a file is truncated.
|
||||||
|
*/
|
||||||
|
public static class TruncateEvent extends Event {
|
||||||
|
private String path;
|
||||||
|
private long fileSize;
|
||||||
|
private long timestamp;
|
||||||
|
|
||||||
|
|
||||||
|
public TruncateEvent(String path, long fileSize, long timestamp) {
|
||||||
|
super(EventType.TRUNCATE);
|
||||||
|
this.path = path;
|
||||||
|
this.fileSize = fileSize;
|
||||||
|
this.timestamp = timestamp;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPath() {
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The size of the truncated file in bytes.
|
||||||
|
*/
|
||||||
|
public long getFileSize() {
|
||||||
|
return fileSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The time when this event occurred, in milliseconds since the epoch.
|
||||||
|
*/
|
||||||
|
public long getTimestamp() {
|
||||||
|
return timestamp;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,6 +41,7 @@ enum EventType {
|
||||||
EVENT_RENAME = 0x3;
|
EVENT_RENAME = 0x3;
|
||||||
EVENT_METADATA = 0x4;
|
EVENT_METADATA = 0x4;
|
||||||
EVENT_UNLINK = 0x5;
|
EVENT_UNLINK = 0x5;
|
||||||
|
EVENT_TRUNCATE = 0x6;
|
||||||
}
|
}
|
||||||
|
|
||||||
message EventProto {
|
message EventProto {
|
||||||
|
@ -87,6 +88,12 @@ message CloseEventProto {
|
||||||
required int64 timestamp = 3;
|
required int64 timestamp = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message TruncateEventProto {
|
||||||
|
required string path = 1;
|
||||||
|
required int64 fileSize = 2;
|
||||||
|
required int64 timestamp = 3;
|
||||||
|
}
|
||||||
|
|
||||||
message AppendEventProto {
|
message AppendEventProto {
|
||||||
required string path = 1;
|
required string path = 1;
|
||||||
optional bool newBlock = 2 [default = false];
|
optional bool newBlock = 2 [default = false];
|
||||||
|
|
|
@ -380,6 +380,9 @@ Release 2.8.0 - UNRELEASED
|
||||||
HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move
|
HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move
|
||||||
progress. (Surendra Singh Lilhore via szetszwo)
|
progress. (Surendra Singh Lilhore via szetszwo)
|
||||||
|
|
||||||
|
HDFS-8742. Inotify: Support event for OP_TRUNCATE.
|
||||||
|
(Surendra Singh Lilhore via aajisaka)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
|
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
|
||||||
|
|
|
@ -2686,6 +2686,12 @@ public class PBHelper {
|
||||||
.timestamp(unlink.getTimestamp())
|
.timestamp(unlink.getTimestamp())
|
||||||
.build());
|
.build());
|
||||||
break;
|
break;
|
||||||
|
case EVENT_TRUNCATE:
|
||||||
|
InotifyProtos.TruncateEventProto truncate =
|
||||||
|
InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
|
||||||
|
events.add(new Event.TruncateEvent(truncate.getPath(),
|
||||||
|
truncate.getFileSize(), truncate.getTimestamp()));
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
throw new RuntimeException("Unexpected inotify event type: " +
|
throw new RuntimeException("Unexpected inotify event type: " +
|
||||||
p.getType());
|
p.getType());
|
||||||
|
@ -2792,6 +2798,17 @@ public class PBHelper {
|
||||||
.setTimestamp(ue.getTimestamp()).build().toByteString()
|
.setTimestamp(ue.getTimestamp()).build().toByteString()
|
||||||
).build());
|
).build());
|
||||||
break;
|
break;
|
||||||
|
case TRUNCATE:
|
||||||
|
Event.TruncateEvent te = (Event.TruncateEvent) e;
|
||||||
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
|
.setType(InotifyProtos.EventType.EVENT_TRUNCATE)
|
||||||
|
.setContents(
|
||||||
|
InotifyProtos.TruncateEventProto.newBuilder()
|
||||||
|
.setPath(te.getPath())
|
||||||
|
.setFileSize(te.getFileSize())
|
||||||
|
.setTimestamp(te.getTimestamp()).build().toByteString()
|
||||||
|
).build());
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
throw new RuntimeException("Unexpected inotify event: " + e);
|
throw new RuntimeException("Unexpected inotify event: " + e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -176,6 +176,10 @@ public class InotifyFSEditLogOpTranslator {
|
||||||
.metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS)
|
.metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS)
|
||||||
.path(saOp.src)
|
.path(saOp.src)
|
||||||
.acls(saOp.aclEntries).build() });
|
.acls(saOp.aclEntries).build() });
|
||||||
|
case OP_TRUNCATE:
|
||||||
|
FSEditLogOp.TruncateOp tOp = (FSEditLogOp.TruncateOp) op;
|
||||||
|
return new EventBatch(op.txid, new Event[] {
|
||||||
|
new Event.TruncateEvent(tOp.src, tOp.newLength, tOp.timestamp) });
|
||||||
default:
|
default:
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -102,6 +102,8 @@ public class TestDFSInotifyEventInputStream {
|
||||||
DFSTestUtil.createFile(fs, new Path("/file"), BLOCK_SIZE, (short) 1, 0L);
|
DFSTestUtil.createFile(fs, new Path("/file"), BLOCK_SIZE, (short) 1, 0L);
|
||||||
DFSTestUtil.createFile(fs, new Path("/file3"), BLOCK_SIZE, (short) 1, 0L);
|
DFSTestUtil.createFile(fs, new Path("/file3"), BLOCK_SIZE, (short) 1, 0L);
|
||||||
DFSTestUtil.createFile(fs, new Path("/file5"), BLOCK_SIZE, (short) 1, 0L);
|
DFSTestUtil.createFile(fs, new Path("/file5"), BLOCK_SIZE, (short) 1, 0L);
|
||||||
|
DFSTestUtil.createFile(fs, new Path("/truncate_file"),
|
||||||
|
BLOCK_SIZE * 2, (short) 1, 0L);
|
||||||
DFSInotifyEventInputStream eis = client.getInotifyEventStream();
|
DFSInotifyEventInputStream eis = client.getInotifyEventStream();
|
||||||
client.rename("/file", "/file4", null); // RenameOp -> RenameEvent
|
client.rename("/file", "/file4", null); // RenameOp -> RenameEvent
|
||||||
client.rename("/file4", "/file2"); // RenameOldOp -> RenameEvent
|
client.rename("/file4", "/file2"); // RenameOldOp -> RenameEvent
|
||||||
|
@ -136,7 +138,8 @@ public class TestDFSInotifyEventInputStream {
|
||||||
"user::rwx,user:foo:rw-,group::r--,other::---", true));
|
"user::rwx,user:foo:rw-,group::r--,other::---", true));
|
||||||
client.removeAcl("/file5"); // SetAclOp -> MetadataUpdateEvent
|
client.removeAcl("/file5"); // SetAclOp -> MetadataUpdateEvent
|
||||||
client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent
|
client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent
|
||||||
|
//TruncateOp -> TruncateEvent
|
||||||
|
client.truncate("/truncate_file", BLOCK_SIZE);
|
||||||
EventBatch batch = null;
|
EventBatch batch = null;
|
||||||
|
|
||||||
// RenameOp
|
// RenameOp
|
||||||
|
@ -354,6 +357,18 @@ public class TestDFSInotifyEventInputStream {
|
||||||
Assert.assertTrue(re3.getSrcPath().equals("/file5"));
|
Assert.assertTrue(re3.getSrcPath().equals("/file5"));
|
||||||
Assert.assertTrue(re.getTimestamp() > 0);
|
Assert.assertTrue(re.getTimestamp() > 0);
|
||||||
|
|
||||||
|
// TruncateOp
|
||||||
|
batch = waitForNextEvents(eis);
|
||||||
|
Assert.assertEquals(1, batch.getEvents().length);
|
||||||
|
txid = checkTxid(batch, txid);
|
||||||
|
Assert
|
||||||
|
.assertTrue(batch.getEvents()[0].getEventType() ==
|
||||||
|
Event.EventType.TRUNCATE);
|
||||||
|
Event.TruncateEvent et = ((Event.TruncateEvent) batch.getEvents()[0]);
|
||||||
|
Assert.assertTrue(et.getPath().equals("/truncate_file"));
|
||||||
|
Assert.assertTrue(et.getFileSize() == BLOCK_SIZE);
|
||||||
|
Assert.assertTrue(et.getTimestamp() > 0);
|
||||||
|
|
||||||
// Returns null when there are no further events
|
// Returns null when there are no further events
|
||||||
Assert.assertTrue(eis.poll() == null);
|
Assert.assertTrue(eis.poll() == null);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue