Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe)

This commit is contained in:
Colin Patrick Mccabe 2014-12-15 10:00:25 -08:00
parent fae3e8614f
commit 6e13fc62e1
6 changed files with 124 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -457,6 +457,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7426. Change nntop JMX format to be a JSON blob. (wang)
HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe)
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

81
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
View File

@ -101,6 +101,7 @@ public abstract class Event {
private FsPermission perms;
private String symlinkTarget;
private boolean overwrite;
private long defaultBlockSize;
public static class Builder {
private INodeType iNodeType;
@ -112,6 +113,7 @@ public abstract class Event {
private FsPermission perms;
private String symlinkTarget;
private boolean overwrite;
private long defaultBlockSize = 0;
public Builder iNodeType(INodeType type) {
this.iNodeType = type;
@ -158,6 +160,11 @@ public abstract class Event {
return this;
}
public Builder defaultBlockSize(long defaultBlockSize) {
this.defaultBlockSize = defaultBlockSize;
return this;
}
public CreateEvent build() {
return new CreateEvent(this);
}
@ -174,6 +181,7 @@ public abstract class Event {
this.perms = b.perms;
this.symlinkTarget = b.symlinkTarget;
this.overwrite = b.overwrite;
this.defaultBlockSize = b.defaultBlockSize;
}
public INodeType getiNodeType() {
@ -220,6 +228,10 @@ public abstract class Event {
public boolean getOverwrite() {
return overwrite;
}
public long getDefaultBlockSize() {
return defaultBlockSize;
}
}
/**
@ -398,11 +410,36 @@ public abstract class Event {
private String dstPath;
private long timestamp;
public RenameEvent(String srcPath, String dstPath, long timestamp) {
super(EventType.RENAME);
public static class Builder {
private String srcPath;
private String dstPath;
private long timestamp;
public Builder srcPath(String srcPath) {
this.srcPath = srcPath;
return this;
}
public Builder dstPath(String dstPath) {
this.dstPath = dstPath;
return this;
}
public Builder timestamp(long timestamp) {
this.timestamp = timestamp;
return this;
}
public RenameEvent build() {
return new RenameEvent(this);
}
}
private RenameEvent(Builder builder) {
super(EventType.RENAME);
this.srcPath = builder.srcPath;
this.dstPath = builder.dstPath;
this.timestamp = builder.timestamp;
}
public String getSrcPath() {
@ -427,9 +464,22 @@ public abstract class Event {
public static class AppendEvent extends Event {
private String path;
public AppendEvent(String path) {
super(EventType.APPEND);
public static class Builder {
private String path;
public Builder path(String path) {
this.path = path;
return this;
}
public AppendEvent build() {
return new AppendEvent(this);
}
}
private AppendEvent(Builder b) {
super(EventType.APPEND);
this.path = b.path;
}
public String getPath() {
@ -444,10 +494,29 @@ public abstract class Event {
private String path;
private long timestamp;
public UnlinkEvent(String path, long timestamp) {
super(EventType.UNLINK);
public static class Builder {
private String path;
private long timestamp;
public Builder path(String path) {
this.path = path;
return this;
}
public Builder timestamp(long timestamp) {
this.timestamp = timestamp;
return this;
}
public UnlinkEvent build() {
return new UnlinkEvent(this);
}
}
private UnlinkEvent(Builder builder) {
super(EventType.UNLINK);
this.path = builder.path;
this.timestamp = builder.timestamp;
}
public String getPath() {

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
View File

@ -2566,6 +2566,7 @@ public class PBHelper {
.replication(create.getReplication())
.symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
create.getSymlinkTarget())
.defaultBlockSize(create.getDefaultBlockSize())
.overwrite(create.getOverwrite()).build());
break;
case EVENT_METADATA:
@ -2592,19 +2593,26 @@ public class PBHelper {
case EVENT_RENAME:
InotifyProtos.RenameEventProto rename =
InotifyProtos.RenameEventProto.parseFrom(p.getContents());
events.add(new Event.RenameEvent(rename.getSrcPath(),
rename.getDestPath(), rename.getTimestamp()));
events.add(new Event.RenameEvent.Builder()
.srcPath(rename.getSrcPath())
.dstPath(rename.getDestPath())
.timestamp(rename.getTimestamp())
.build());
break;
case EVENT_APPEND:
InotifyProtos.AppendEventProto reopen =
InotifyProtos.AppendEventProto.parseFrom(p.getContents());
events.add(new Event.AppendEvent(reopen.getPath()));
events.add(new Event.AppendEvent.Builder()
.path(reopen.getPath())
.build());
break;
case EVENT_UNLINK:
InotifyProtos.UnlinkEventProto unlink =
InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
events.add(new Event.UnlinkEvent(unlink.getPath(),
unlink.getTimestamp()));
events.add(new Event.UnlinkEvent.Builder()
.path(unlink.getPath())
.timestamp(unlink.getTimestamp())
.build());
break;
default:
throw new RuntimeException("Unexpected inotify event type: " +
@ -2650,6 +2658,7 @@ public class PBHelper {
.setReplication(ce2.getReplication())
.setSymlinkTarget(ce2.getSymlinkTarget() == null ?
"" : ce2.getSymlinkTarget())
.setDefaultBlockSize(ce2.getDefaultBlockSize())
.setOverwrite(ce2.getOverwrite()).build().toByteString()
).build());
break;

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
View File

@ -53,10 +53,13 @@ public class InotifyFSEditLogOpTranslator {
.groupName(addOp.permissions.getGroupName())
.perms(addOp.permissions.getPermission())
.overwrite(addOp.overwrite)
.defaultBlockSize(addOp.blockSize)
.iNodeType(Event.CreateEvent.INodeType.FILE).build() });
} else {
} else { // append
return new EventBatch(op.txid,
new Event[] { new Event.AppendEvent(addOp.path) });
new Event[]{new Event.AppendEvent.Builder()
.path(addOp.path)
.build()});
}
case OP_CLOSE:
FSEditLogOp.CloseOp cOp = (FSEditLogOp.CloseOp) op;
@ -72,25 +75,40 @@ public class InotifyFSEditLogOpTranslator {
case OP_CONCAT_DELETE:
FSEditLogOp.ConcatDeleteOp cdOp = (FSEditLogOp.ConcatDeleteOp) op;
List<Event> events = Lists.newArrayList();
events.add(new Event.AppendEvent(cdOp.trg));
events.add(new Event.AppendEvent.Builder()
.path(cdOp.trg)
.build());
for (String src : cdOp.srcs) {
events.add(new Event.UnlinkEvent(src, cdOp.timestamp));
events.add(new Event.UnlinkEvent.Builder()
.path(src)
.timestamp(cdOp.timestamp)
.build());
}
events.add(new Event.CloseEvent(cdOp.trg, -1, cdOp.timestamp));
return new EventBatch(op.txid, events.toArray(new Event[0]));
case OP_RENAME_OLD:
FSEditLogOp.RenameOldOp rnOpOld = (FSEditLogOp.RenameOldOp) op;
return new EventBatch(op.txid, new Event[] {
new Event.RenameEvent(rnOpOld.src,
rnOpOld.dst, rnOpOld.timestamp) });
new Event.RenameEvent.Builder()
.srcPath(rnOpOld.src)
.dstPath(rnOpOld.dst)
.timestamp(rnOpOld.timestamp)
.build() });
case OP_RENAME:
FSEditLogOp.RenameOp rnOp = (FSEditLogOp.RenameOp) op;
return new EventBatch(op.txid, new Event[] {
new Event.RenameEvent(rnOp.src, rnOp.dst, rnOp.timestamp) });
new Event.RenameEvent.Builder()
.srcPath(rnOp.src)
.dstPath(rnOp.dst)
.timestamp(rnOp.timestamp)
.build() });
case OP_DELETE:
FSEditLogOp.DeleteOp delOp = (FSEditLogOp.DeleteOp) op;
return new EventBatch(op.txid, new Event[] {
new Event.UnlinkEvent(delOp.path, delOp.timestamp) });
new Event.UnlinkEvent.Builder()
.path(delOp.path)
.timestamp(delOp.timestamp)
.build() });
case OP_MKDIR:
FSEditLogOp.MkdirOp mkOp = (FSEditLogOp.MkdirOp) op;
return new EventBatch(op.txid,

1
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
View File

@ -78,6 +78,7 @@ message CreateEventProto {
optional int32 replication = 7;
optional string symlinkTarget = 8;
optional bool overwrite = 9;
optional int64 defaultBlockSize = 10 [default=0];
}
message CloseEventProto {

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
View File

@ -170,6 +170,7 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(ce.getReplication() > 0);
Assert.assertTrue(ce.getSymlinkTarget() == null);
Assert.assertTrue(ce.getOverwrite());
Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize());
// CloseOp
batch = waitForNextEvents(eis);
@ -186,7 +187,8 @@ public class TestDFSInotifyEventInputStream {
Assert.assertEquals(1, batch.getEvents().length);
txid = checkTxid(batch, txid);
Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.APPEND);
Assert.assertTrue(((Event.AppendEvent) batch.getEvents()[0]).getPath().equals("/file2"));
Event.AppendEvent append2 = (Event.AppendEvent)batch.getEvents()[0];
Assert.assertEquals("/file2", append2.getPath());
// CloseOp
batch = waitForNextEvents(eis);