HDFS-6492. Support create-time xattrs and atomically setting multiple xattrs. (wang)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1603978 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Wang 2014-06-19 17:55:01 +00:00
parent b61992e807
commit 17a6d23967
13 changed files with 623 additions and 263 deletions

View File

@ -481,6 +481,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6374. setXAttr should require the user to be the owner of the file
or directory (Charles Lamb via wang)
HDFS-6492. Support create-time xattrs and atomically setting multiple
xattrs. (wang)
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES

View File

@ -19,6 +19,8 @@ package org.apache.hadoop.fs;
import java.util.Arrays;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
/**
@ -105,40 +107,45 @@ public class XAttr {
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((name == null) ? 0 : name.hashCode());
result = prime * result + ((ns == null) ? 0 : ns.hashCode());
result = prime * result + Arrays.hashCode(value);
return result;
return new HashCodeBuilder(811, 67)
.append(name)
.append(ns)
.append(value)
.toHashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
if (obj == null) { return false; }
if (obj == this) { return true; }
if (obj.getClass() != getClass()) {
return false;
}
if (getClass() != obj.getClass()) {
XAttr rhs = (XAttr) obj;
return new EqualsBuilder()
.append(ns, rhs.ns)
.append(name, rhs.name)
.append(value, rhs.value)
.isEquals();
}
/**
* Similar to {@link #equals(Object)}, except ignores the XAttr value.
*
* @param obj to compare equality
* @return if the XAttrs are equal, ignoring the XAttr value
*/
public boolean equalsIgnoreValue(Object obj) {
if (obj == null) { return false; }
if (obj == this) { return true; }
if (obj.getClass() != getClass()) {
return false;
}
XAttr other = (XAttr) obj;
if (name == null) {
if (other.name != null) {
return false;
}
} else if (!name.equals(other.name)) {
return false;
}
if (ns != other.ns) {
return false;
}
if (!Arrays.equals(value, other.value)) {
return false;
}
return true;
XAttr rhs = (XAttr) obj;
return new EqualsBuilder()
.append(ns, rhs.ns)
.append(name, rhs.name)
.isEquals();
}
@Override

View File

@ -2092,6 +2092,9 @@ public class PBHelper {
public static List<XAttrProto> convertXAttrProto(
List<XAttr> xAttrSpec) {
if (xAttrSpec == null) {
return Lists.newArrayListWithCapacity(0);
}
ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity(
xAttrSpec.size());
for (XAttr a : xAttrSpec) {

View File

@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.ListIterator;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.HadoopIllegalArgumentException;
@ -295,6 +296,7 @@ public class FSDirectory implements Closeable {
String path,
PermissionStatus permissions,
List<AclEntry> aclEntries,
List<XAttr> xAttrs,
short replication,
long modificationTime,
long atime,
@ -321,6 +323,10 @@ public class FSDirectory implements Closeable {
AclStorage.updateINodeAcl(newNode, aclEntries,
Snapshot.CURRENT_STATE_ID);
}
if (xAttrs != null) {
XAttrStorage.updateINodeXAttrs(newNode, xAttrs,
Snapshot.CURRENT_STATE_ID);
}
return newNode;
}
} catch (IOException e) {
@ -2565,103 +2571,165 @@ public class FSDirectory implements Closeable {
}
}
void removeXAttr(String src, XAttr xAttr) throws IOException {
/**
* Removes a list of XAttrs from an inode at a path.
*
* @param src path of inode
* @param toRemove XAttrs to be removed
* @return List of XAttrs that were removed
* @throws IOException if the inode does not exist, if quota is exceeded
*/
List<XAttr> removeXAttrs(final String src, final List<XAttr> toRemove)
throws IOException {
writeLock();
try {
XAttr removedXAttr = unprotectedRemoveXAttr(src, xAttr);
if (removedXAttr != null) {
fsImage.getEditLog().logRemoveXAttr(src, removedXAttr);
} else {
NameNode.stateChangeLog.info("DIR* FSDirectory.removeXAttr: XAttr " +
XAttrHelper.getPrefixName(xAttr) +
" does not exist on the path " + src);
}
return unprotectedRemoveXAttrs(src, toRemove);
} finally {
writeUnlock();
}
}
XAttr unprotectedRemoveXAttr(String src,
XAttr xAttr) throws IOException {
List<XAttr> unprotectedRemoveXAttrs(final String src,
final List<XAttr> toRemove) throws IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> newXAttrs = filterINodeXAttr(existingXAttrs, xAttr);
List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove,
removedXAttrs);
if (existingXAttrs.size() != newXAttrs.size()) {
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
return xAttr;
return removedXAttrs;
}
return null;
}
List<XAttr> filterINodeXAttr(List<XAttr> existingXAttrs,
XAttr xAttr) throws QuotaExceededException {
if (existingXAttrs == null || existingXAttrs.isEmpty()) {
/**
* Filter XAttrs from a list of existing XAttrs. Removes matched XAttrs from
* toFilter and puts them into filtered. Upon completion,
* toFilter contains the filter XAttrs that were not found, while
* fitleredXAttrs contains the XAttrs that were found.
*
* @param existingXAttrs Existing XAttrs to be filtered
* @param toFilter XAttrs to filter from the existing XAttrs
* @param filtered Return parameter, XAttrs that were filtered
* @return List of XAttrs that does not contain filtered XAttrs
*/
@VisibleForTesting
List<XAttr> filterINodeXAttrs(final List<XAttr> existingXAttrs,
final List<XAttr> toFilter, final List<XAttr> filtered) {
if (existingXAttrs == null || existingXAttrs.isEmpty() ||
toFilter == null || toFilter.isEmpty()) {
return existingXAttrs;
}
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(existingXAttrs.size());
// Populate a new list with XAttrs that pass the filter
List<XAttr> newXAttrs =
Lists.newArrayListWithCapacity(existingXAttrs.size());
for (XAttr a : existingXAttrs) {
if (!(a.getNameSpace() == xAttr.getNameSpace()
&& a.getName().equals(xAttr.getName()))) {
xAttrs.add(a);
boolean add = true;
for (ListIterator<XAttr> it = toFilter.listIterator(); it.hasNext()
;) {
XAttr filter = it.next();
if (a.equalsIgnoreValue(filter)) {
add = false;
it.remove();
filtered.add(filter);
break;
}
}
if (add) {
newXAttrs.add(a);
}
}
return xAttrs;
return newXAttrs;
}
void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache) throws IOException {
void setXAttrs(final String src, final List<XAttr> xAttrs,
final EnumSet<XAttrSetFlag> flag) throws IOException {
writeLock();
try {
unprotectedSetXAttr(src, xAttr, flag);
fsImage.getEditLog().logSetXAttr(src, xAttr, logRetryCache);
unprotectedSetXAttrs(src, xAttrs, flag);
} finally {
writeUnlock();
}
}
void unprotectedSetXAttr(String src, XAttr xAttr,
EnumSet<XAttrSetFlag> flag) throws IOException {
void unprotectedSetXAttrs(final String src, final List<XAttr> xAttrs,
final EnumSet<XAttrSetFlag> flag)
throws QuotaExceededException, IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> newXAttrs = setINodeXAttr(existingXAttrs, xAttr, flag);
List<XAttr> newXAttrs = setINodeXAttrs(existingXAttrs, xAttrs, flag);
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
}
List<XAttr> setINodeXAttr(List<XAttr> existingXAttrs, XAttr xAttr,
EnumSet<XAttrSetFlag> flag) throws QuotaExceededException, IOException {
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(
existingXAttrs != null ? existingXAttrs.size() + 1 : 1);
List<XAttr> setINodeXAttrs(final List<XAttr> existingXAttrs,
final List<XAttr> toSet, final EnumSet<XAttrSetFlag> flag)
throws IOException {
// Check for duplicate XAttrs in toSet
// We need to use a custom comparator, so using a HashSet is not suitable
for (int i = 0; i < toSet.size(); i++) {
for (int j = i + 1; j < toSet.size(); j++) {
if (toSet.get(i).equalsIgnoreValue(toSet.get(j))) {
throw new IOException("Cannot specify the same XAttr to be set " +
"more than once");
}
}
}
// Count the current number of user-visible XAttrs for limit checking
int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
// The XAttr list is copied to an exactly-sized array when it's stored,
// so there's no need to size it precisely here.
int newSize = (existingXAttrs != null) ? existingXAttrs.size() : 0;
newSize += toSet.size();
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(newSize);
// Check if the XAttr already exists to validate with the provided flag
for (XAttr xAttr: toSet) {
boolean exist = false;
if (existingXAttrs != null) {
for (XAttr a: existingXAttrs) {
if ((a.getNameSpace() == xAttr.getNameSpace()
&& a.getName().equals(xAttr.getName()))) {
for (XAttr a : existingXAttrs) {
if (a.equalsIgnoreValue(xAttr)) {
exist = true;
} else {
xAttrs.add(a);
if (isUserVisible(a)) {
userVisibleXAttrsNum++;
break;
}
}
}
}
XAttrSetFlag.validate(xAttr.getName(), exist, flag);
// add the new XAttr since it passed validation
xAttrs.add(xAttr);
if (isUserVisible(xAttr)) {
userVisibleXAttrsNum++;
}
}
// Add the existing xattrs back in, if they weren't already set
if (existingXAttrs != null) {
for (XAttr existing : existingXAttrs) {
boolean alreadySet = false;
for (XAttr set : toSet) {
if (set.equalsIgnoreValue(existing)) {
alreadySet = true;
break;
}
}
if (!alreadySet) {
xAttrs.add(existing);
if (isUserVisible(existing)) {
userVisibleXAttrsNum++;
}
}
}
}
if (userVisibleXAttrsNum > inodeXAttrsLimit) {
throw new IOException("Cannot add additional XAttr to inode, "

View File

@ -700,12 +700,19 @@ public class FSEditLog implements LogsPurgeable {
.setBlocks(newNode.getBlocks())
.setPermissionStatus(permissions)
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
.setClientMachine(newNode.getFileUnderConstructionFeature().getClientMachine());
.setClientMachine(
newNode.getFileUnderConstructionFeature().getClientMachine());
AclFeature f = newNode.getAclFeature();
if (f != null) {
op.setAclEntries(AclStorage.readINodeLogicalAcl(newNode));
}
XAttrFeature x = newNode.getXAttrFeature();
if (x != null) {
op.setXAttrs(x.getXAttrs());
}
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
@ -761,6 +768,11 @@ public class FSEditLog implements LogsPurgeable {
if (f != null) {
op.setAclEntries(AclStorage.readINodeLogicalAcl(newNode));
}
XAttrFeature x = newNode.getXAttrFeature();
if (x != null) {
op.setXAttrs(x.getXAttrs());
}
logEdit(op);
}
@ -1054,18 +1066,18 @@ public class FSEditLog implements LogsPurgeable {
logEdit(op);
}
void logSetXAttr(String src, XAttr xAttr, boolean toLogRpcIds) {
void logSetXAttrs(String src, List<XAttr> xAttrs, boolean toLogRpcIds) {
final SetXAttrOp op = SetXAttrOp.getInstance();
op.src = src;
op.xAttr = xAttr;
op.xAttrs = xAttrs;
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
void logRemoveXAttr(String src, XAttr xAttr) {
void logRemoveXAttrs(String src, List<XAttr> xAttrs) {
final RemoveXAttrOp op = RemoveXAttrOp.getInstance();
op.src = src;
op.xAttr = xAttr;
op.xAttrs = xAttrs;
logEdit(op);
}

View File

@ -356,6 +356,7 @@ public class FSEditLogLoader {
lastInodeId);
newFile = fsDir.unprotectedAddFile(inodeId,
path, addCloseOp.permissions, addCloseOp.aclEntries,
addCloseOp.xAttrs,
replication, addCloseOp.mtime, addCloseOp.atime,
addCloseOp.blockSize, true, addCloseOp.clientName,
addCloseOp.clientMachine);
@ -808,7 +809,7 @@ public class FSEditLogLoader {
}
case OP_SET_XATTR: {
SetXAttrOp setXAttrOp = (SetXAttrOp) op;
fsDir.unprotectedSetXAttr(setXAttrOp.src, setXAttrOp.xAttr,
fsDir.unprotectedSetXAttrs(setXAttrOp.src, setXAttrOp.xAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(setXAttrOp.rpcClientId, setXAttrOp.rpcCallId);
@ -817,7 +818,8 @@ public class FSEditLogLoader {
}
case OP_REMOVE_XATTR: {
RemoveXAttrOp removeXAttrOp = (RemoveXAttrOp) op;
fsDir.unprotectedRemoveXAttr(removeXAttrOp.src, removeXAttrOp.xAttr);
fsDir.unprotectedRemoveXAttrs(removeXAttrOp.src,
removeXAttrOp.xAttrs);
break;
}
default:

View File

@ -384,6 +384,16 @@ public abstract class FSEditLogOp {
}
}
private static List<XAttr> readXAttrsFromEditLog(DataInputStream in,
int logVersion) throws IOException {
if (!NameNodeLayoutVersion.supports(NameNodeLayoutVersion.Feature.XATTRS,
logVersion)) {
return null;
}
XAttrEditLogProto proto = XAttrEditLogProto.parseDelimitedFrom(in);
return PBHelper.convertXAttrs(proto.getXAttrsList());
}
@SuppressWarnings("unchecked")
static abstract class AddCloseOp extends FSEditLogOp implements BlockListUpdatingOp {
int length;
@ -396,6 +406,7 @@ public abstract class FSEditLogOp {
Block[] blocks;
PermissionStatus permissions;
List<AclEntry> aclEntries;
List<XAttr> xAttrs;
String clientName;
String clientMachine;
@ -463,6 +474,11 @@ public abstract class FSEditLogOp {
return (T)this;
}
<T extends AddCloseOp> T setXAttrs(List<XAttr> xAttrs) {
this.xAttrs = xAttrs;
return (T)this;
}
<T extends AddCloseOp> T setClientName(String clientName) {
this.clientName = clientName;
return (T)this;
@ -486,6 +502,9 @@ public abstract class FSEditLogOp {
if (this.opCode == OP_ADD) {
AclEditLogUtil.write(aclEntries, out);
XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
// write clientId and callId
@ -548,9 +567,9 @@ public abstract class FSEditLogOp {
this.blocks = readBlocks(in, logVersion);
this.permissions = PermissionStatus.read(in);
// clientname, clientMachine and block locations of last block.
if (this.opCode == OP_ADD) {
aclEntries = AclEditLogUtil.read(in, logVersion);
this.xAttrs = readXAttrsFromEditLog(in, logVersion);
this.clientName = FSImageSerialization.readString(in);
this.clientMachine = FSImageSerialization.readString(in);
// read clientId and callId
@ -1345,6 +1364,7 @@ public abstract class FSEditLogOp {
long timestamp;
PermissionStatus permissions;
List<AclEntry> aclEntries;
List<XAttr> xAttrs;
private MkdirOp() {
super(OP_MKDIR);
@ -1379,6 +1399,11 @@ public abstract class FSEditLogOp {
return this;
}
MkdirOp setXAttrs(List<XAttr> xAttrs) {
this.xAttrs = xAttrs;
return this;
}
@Override
public
void writeFields(DataOutputStream out) throws IOException {
@ -1388,6 +1413,9 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeLong(timestamp, out); // atime, unused at this
permissions.write(out);
AclEditLogUtil.write(aclEntries, out);
XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
}
@Override
@ -1432,6 +1460,8 @@ public abstract class FSEditLogOp {
this.permissions = PermissionStatus.read(in);
aclEntries = AclEditLogUtil.read(in, logVersion);
xAttrs = readXAttrsFromEditLog(in, logVersion);
}
@Override
@ -1453,6 +1483,8 @@ public abstract class FSEditLogOp {
builder.append(opCode);
builder.append(", txid=");
builder.append(txid);
builder.append(", xAttrs=");
builder.append(xAttrs);
builder.append("]");
return builder.toString();
}
@ -1470,6 +1502,9 @@ public abstract class FSEditLogOp {
if (aclEntries != null) {
appendAclEntriesToXml(contentHandler, aclEntries);
}
if (xAttrs != null) {
appendXAttrsToXml(contentHandler, xAttrs);
}
}
@Override void fromXml(Stanza st) throws InvalidXmlException {
@ -1479,6 +1514,7 @@ public abstract class FSEditLogOp {
this.timestamp = Long.parseLong(st.getValue("TIMESTAMP"));
this.permissions = permissionStatusFromXml(st);
aclEntries = readAclEntriesFromXml(st);
xAttrs = readXAttrsFromXml(st);
}
}
@ -3501,7 +3537,7 @@ public abstract class FSEditLogOp {
}
static class RemoveXAttrOp extends FSEditLogOp {
XAttr xAttr;
List<XAttr> xAttrs;
String src;
private RemoveXAttrOp() {
@ -3516,7 +3552,7 @@ public abstract class FSEditLogOp {
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
xAttr = PBHelper.convertXAttr(p.getXAttr());
xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
}
@Override
@ -3525,25 +3561,25 @@ public abstract class FSEditLogOp {
if (src != null) {
b.setSrc(src);
}
b.setXAttr(PBHelper.convertXAttrProto(xAttr));
b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
}
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
XMLUtils.addSaxString(contentHandler, "SRC", src);
appendXAttrToXml(contentHandler, xAttr);
appendXAttrsToXml(contentHandler, xAttrs);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
src = st.getValue("SRC");
xAttr = readXAttrFromXml(st);
xAttrs = readXAttrsFromXml(st);
}
}
static class SetXAttrOp extends FSEditLogOp {
XAttr xAttr;
List<XAttr> xAttrs;
String src;
private SetXAttrOp() {
@ -3558,7 +3594,7 @@ public abstract class FSEditLogOp {
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
xAttr = PBHelper.convertXAttr(p.getXAttr());
xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
readRpcIds(in, logVersion);
}
@ -3568,7 +3604,7 @@ public abstract class FSEditLogOp {
if (src != null) {
b.setSrc(src);
}
b.setXAttr(PBHelper.convertXAttrProto(xAttr));
b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
// clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
@ -3577,14 +3613,14 @@ public abstract class FSEditLogOp {
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
XMLUtils.addSaxString(contentHandler, "SRC", src);
appendXAttrToXml(contentHandler, xAttr);
appendXAttrsToXml(contentHandler, xAttrs);
appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
src = st.getValue("SRC");
xAttr = readXAttrFromXml(st);
xAttrs = readXAttrsFromXml(st);
readRpcIdsFromXml(st);
}
}
@ -4205,8 +4241,9 @@ public abstract class FSEditLogOp {
return aclEntries;
}
private static void appendXAttrToXml(ContentHandler contentHandler,
XAttr xAttr) throws SAXException {
private static void appendXAttrsToXml(ContentHandler contentHandler,
List<XAttr> xAttrs) throws SAXException {
for (XAttr xAttr: xAttrs) {
contentHandler.startElement("", "", "XATTR", new AttributesImpl());
XMLUtils.addSaxString(contentHandler, "NAMESPACE",
xAttr.getNameSpace().toString());
@ -4221,14 +4258,17 @@ public abstract class FSEditLogOp {
}
contentHandler.endElement("", "", "XATTR");
}
}
private static XAttr readXAttrFromXml(Stanza st)
private static List<XAttr> readXAttrsFromXml(Stanza st)
throws InvalidXmlException {
if (!st.hasChildren("XATTR")) {
return null;
}
Stanza a = st.getChildren("XATTR").get(0);
List<Stanza> stanzas = st.getChildren("XATTR");
List<XAttr> xattrs = Lists.newArrayListWithCapacity(stanzas.size());
for (Stanza a: stanzas) {
XAttr.Builder builder = new XAttr.Builder();
builder.setNameSpace(XAttr.NameSpace.valueOf(a.getValue("NAMESPACE"))).
setName(a.getValue("NAME"));
@ -4240,6 +4280,8 @@ public abstract class FSEditLogOp {
throw new InvalidXmlException(e.toString());
}
}
return builder.build();
xattrs.add(builder.build());
}
return xattrs;
}
}

View File

@ -8193,7 +8193,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
checkOwner(pc, src);
checkPathAccess(pc, src, FsAction.WRITE);
}
dir.setXAttr(src, xAttr, flag, logRetryCache);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
dir.setXAttrs(src, xAttrs, flag);
getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@ -8314,7 +8317,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
checkPathAccess(pc, src, FsAction.WRITE);
}
dir.removeXAttr(src, xAttr);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
List<XAttr> removedXAttrs = dir.removeXAttrs(src, xAttrs);
if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
getEditLog().logRemoveXAttrs(src, removedXAttrs);
}
resultingStat = getAuditFileInfo(src, false);
} catch (AccessControlException e) {
logAuditEvent(false, "removeXAttr", src);

View File

@ -35,8 +35,8 @@ message XAttrProto {
}
message XAttrEditLogProto {
required string src = 1;
optional XAttrProto xAttr = 2;
optional string src = 1;
repeated XAttrProto xAttrs = 2;
}
enum XAttrSetFlagProto {

View File

@ -24,7 +24,9 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.EnumSet;
import java.util.List;
import java.util.Random;
import com.google.common.collect.ImmutableList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@ -44,6 +46,11 @@ import org.junit.Test;
import com.google.common.collect.Lists;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
* Test {@link FSDirectory}, the in-memory namespace tree.
*/
@ -73,6 +80,10 @@ public class TestFSDirectory {
private DistributedFileSystem hdfs;
private static final int numGeneratedXAttrs = 256;
private static final ImmutableList<XAttr> generatedXAttrs =
ImmutableList.copyOf(generateXAttrs(numGeneratedXAttrs));
@Before
public void setUp() throws Exception {
conf = new Configuration();
@ -119,9 +130,10 @@ public class TestFSDirectory {
for(; (line = in.readLine()) != null; ) {
line = line.trim();
if (!line.isEmpty() && !line.contains("snapshot")) {
Assert.assertTrue("line=" + line,
assertTrue("line=" + line,
line.startsWith(INodeDirectory.DUMPTREE_LAST_ITEM)
|| line.startsWith(INodeDirectory.DUMPTREE_EXCEPT_LAST_ITEM));
|| line.startsWith(INodeDirectory.DUMPTREE_EXCEPT_LAST_ITEM)
);
checkClassName(line);
}
}
@ -166,7 +178,7 @@ public class TestFSDirectory {
int i = line.lastIndexOf('(');
int j = line.lastIndexOf('@');
final String classname = line.substring(i+1, j);
Assert.assertTrue(classname.startsWith(INodeFile.class.getSimpleName())
assertTrue(classname.startsWith(INodeFile.class.getSimpleName())
|| classname.startsWith(INodeDirectory.class.getSimpleName()));
}
@ -183,22 +195,185 @@ public class TestFSDirectory {
// Adding a system namespace xAttr, isn't affected by inode xAttrs limit.
XAttr newXAttr = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.SYSTEM).
setName("a3").setValue(new byte[]{0x33, 0x33, 0x33}).build();
List<XAttr> xAttrs = fsdir.setINodeXAttr(existingXAttrs, newXAttr,
List<XAttr> newXAttrs = Lists.newArrayListWithCapacity(1);
newXAttrs.add(newXAttr);
List<XAttr> xAttrs = fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
Assert.assertEquals(xAttrs.size(), 3);
assertEquals(xAttrs.size(), 3);
// Adding a trusted namespace xAttr, is affected by inode xAttrs limit.
XAttr newXAttr1 = (new XAttr.Builder()).setNameSpace(
XAttr.NameSpace.TRUSTED).setName("a4").
setValue(new byte[]{0x34, 0x34, 0x34}).build();
newXAttrs.set(0, newXAttr1);
try {
fsdir.setINodeXAttr(existingXAttrs, newXAttr1,
fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
Assert.fail("Setting user visable xattr on inode should fail if " +
fail("Setting user visible xattr on inode should fail if " +
"reaching limit.");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("Cannot add additional XAttr " +
"to inode, would exceed limit", e);
}
}
/**
* Verify that the first <i>num</i> generatedXAttrs are present in
* newXAttrs.
*/
private static void verifyXAttrsPresent(List<XAttr> newXAttrs,
final int num) {
assertEquals("Unexpected number of XAttrs after multiset", num,
newXAttrs.size());
for (int i=0; i<num; i++) {
XAttr search = generatedXAttrs.get(i);
assertTrue("Did not find set XAttr " + search + " + after multiset",
newXAttrs.contains(search));
}
}
private static List<XAttr> generateXAttrs(final int numXAttrs) {
List<XAttr> generatedXAttrs = Lists.newArrayListWithCapacity(numXAttrs);
for (int i=0; i<numXAttrs; i++) {
XAttr xAttr = (new XAttr.Builder())
.setNameSpace(XAttr.NameSpace.SYSTEM)
.setName("a" + i)
.setValue(new byte[] { (byte) i, (byte) (i + 1), (byte) (i + 2) })
.build();
generatedXAttrs.add(xAttr);
}
return generatedXAttrs;
}
/**
* Test setting and removing multiple xattrs via single operations
*/
@Test(timeout=300000)
public void testXAttrMultiSetRemove() throws Exception {
List<XAttr> existingXAttrs = Lists.newArrayListWithCapacity(0);
// Keep adding a random number of xattrs and verifying until exhausted
final Random rand = new Random(0xFEEDA);
int numExpectedXAttrs = 0;
while (numExpectedXAttrs < numGeneratedXAttrs) {
LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
final int numToAdd = rand.nextInt(5)+1;
List<XAttr> toAdd = Lists.newArrayListWithCapacity(numToAdd);
for (int i = 0; i < numToAdd; i++) {
if (numExpectedXAttrs >= numGeneratedXAttrs) {
break;
}
toAdd.add(generatedXAttrs.get(numExpectedXAttrs));
numExpectedXAttrs++;
}
LOG.info("Attempting to add " + toAdd.size() + " XAttrs");
for (int i = 0; i < toAdd.size(); i++) {
LOG.info("Will add XAttr " + toAdd.get(i));
}
List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE));
verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
existingXAttrs = newXAttrs;
}
// Keep removing a random number of xattrs and verifying until all gone
while (numExpectedXAttrs > 0) {
LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
final int numToRemove = rand.nextInt(5)+1;
List<XAttr> toRemove = Lists.newArrayListWithCapacity(numToRemove);
for (int i = 0; i < numToRemove; i++) {
if (numExpectedXAttrs == 0) {
break;
}
toRemove.add(generatedXAttrs.get(numExpectedXAttrs-1));
numExpectedXAttrs--;
}
final int expectedNumToRemove = toRemove.size();
LOG.info("Attempting to remove " + expectedNumToRemove + " XAttrs");
List<XAttr> removedXAttrs = Lists.newArrayList();
List<XAttr> newXAttrs = fsdir.filterINodeXAttrs(existingXAttrs,
toRemove, removedXAttrs);
assertEquals("Unexpected number of removed XAttrs",
expectedNumToRemove, removedXAttrs.size());
verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
existingXAttrs = newXAttrs;
}
}
@Test(timeout=300000)
public void testXAttrMultiAddRemoveErrors() throws Exception {
// Test that the same XAttr can not be multiset twice
List<XAttr> existingXAttrs = Lists.newArrayList();
List<XAttr> toAdd = Lists.newArrayList();
toAdd.add(generatedXAttrs.get(0));
toAdd.add(generatedXAttrs.get(1));
toAdd.add(generatedXAttrs.get(2));
toAdd.add(generatedXAttrs.get(0));
try {
fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
.CREATE));
fail("Specified the same xattr to be set twice");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("Cannot specify the same " +
"XAttr to be set", e);
}
// Test that CREATE and REPLACE flags are obeyed
toAdd.remove(generatedXAttrs.get(0));
existingXAttrs.add(generatedXAttrs.get(0));
try {
fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
.CREATE));
fail("Set XAttr that is already set without REPLACE flag");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("already exists", e);
}
try {
fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
.REPLACE));
fail("Set XAttr that does not exist without the CREATE flag");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("does not exist", e);
}
// Sanity test for CREATE
toAdd.remove(generatedXAttrs.get(0));
List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE));
assertEquals("Unexpected toAdd size", 2, toAdd.size());
for (XAttr x : toAdd) {
assertTrue("Did not find added XAttr " + x, newXAttrs.contains(x));
}
existingXAttrs = newXAttrs;
// Sanity test for REPLACE
toAdd = Lists.newArrayList();
for (int i=0; i<3; i++) {
XAttr xAttr = (new XAttr.Builder())
.setNameSpace(XAttr.NameSpace.SYSTEM)
.setName("a" + i)
.setValue(new byte[] { (byte) (i*2) })
.build();
toAdd.add(xAttr);
}
newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.REPLACE));
assertEquals("Unexpected number of new XAttrs", 3, newXAttrs.size());
for (int i=0; i<3; i++) {
assertArrayEquals("Unexpected XAttr value",
new byte[] {(byte)(i*2)}, newXAttrs.get(i).getValue());
}
existingXAttrs = newXAttrs;
// Sanity test for CREATE+REPLACE
toAdd = Lists.newArrayList();
for (int i=0; i<4; i++) {
toAdd.add(generatedXAttrs.get(i));
}
newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
verifyXAttrsPresent(newXAttrs, 4);
}
}

View File

@ -249,10 +249,10 @@ public class TestXAttrWithSnapshot {
private static void doSnapshotRootRemovalAssertions(Path path,
Path snapshotPath) throws Exception {
Map<String, byte[]> xattrs = hdfs.getXAttrs(path);
Assert.assertEquals(xattrs.size(), 0);
Assert.assertEquals(0, xattrs.size());
xattrs = hdfs.getXAttrs(snapshotPath);
Assert.assertEquals(xattrs.size(), 2);
Assert.assertEquals(2, xattrs.size());
Assert.assertArrayEquals(value1, xattrs.get(name1));
Assert.assertArrayEquals(value2, xattrs.get(name2));
}

View File

@ -13,8 +13,8 @@
<TXID>2</TXID>
<DELEGATION_KEY>
<KEY_ID>1</KEY_ID>
<EXPIRY_DATE>1394849922137</EXPIRY_DATE>
<KEY>37e1a64049bbef35</KEY>
<EXPIRY_DATE>1403590428625</EXPIRY_DATE>
<KEY>16f34bfba67b2552</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@ -24,8 +24,8 @@
<TXID>3</TXID>
<DELEGATION_KEY>
<KEY_ID>2</KEY_ID>
<EXPIRY_DATE>1394849922140</EXPIRY_DATE>
<KEY>7c0bf5039242fc54</KEY>
<EXPIRY_DATE>1403590428631</EXPIRY_DATE>
<KEY>dbe6282854469833</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@ -37,18 +37,18 @@
<INODEID>16386</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158722811</MTIME>
<ATIME>1394158722811</ATIME>
<MTIME>1402899229669</MTIME>
<ATIME>1402899229669</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>6</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>8</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -59,13 +59,13 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158722832</MTIME>
<ATIME>1394158722811</ATIME>
<MTIME>1402899229711</MTIME>
<ATIME>1402899229669</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@ -78,9 +78,9 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
<TIMESTAMP>1394158722836</TIMESTAMP>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>8</RPC_CALLID>
<TIMESTAMP>1402899229718</TIMESTAMP>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>10</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -89,9 +89,9 @@
<TXID>7</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_moved</PATH>
<TIMESTAMP>1394158722842</TIMESTAMP>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>9</RPC_CALLID>
<TIMESTAMP>1402899229730</TIMESTAMP>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>11</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -101,9 +101,9 @@
<LENGTH>0</LENGTH>
<INODEID>16387</INODEID>
<PATH>/directory_mkdir</PATH>
<TIMESTAMP>1394158722848</TIMESTAMP>
<TIMESTAMP>1402899229748</TIMESTAMP>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>493</MODE>
</PERMISSION_STATUS>
@ -136,8 +136,8 @@
<TXID>12</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>14</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>16</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -147,8 +147,8 @@
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
<SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>15</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>17</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -157,8 +157,8 @@
<TXID>14</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>16</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>18</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -169,18 +169,18 @@
<INODEID>16388</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158722872</MTIME>
<ATIME>1394158722872</ATIME>
<MTIME>1402899229871</MTIME>
<ATIME>1402899229871</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>17</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>19</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -191,13 +191,13 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158722874</MTIME>
<ATIME>1394158722872</ATIME>
<MTIME>1402899229881</MTIME>
<ATIME>1402899229871</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@ -253,10 +253,10 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
<TIMESTAMP>1394158722890</TIMESTAMP>
<TIMESTAMP>1402899229963</TIMESTAMP>
<OPTIONS>NONE</OPTIONS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>24</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>26</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -267,18 +267,18 @@
<INODEID>16389</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158722895</MTIME>
<ATIME>1394158722895</ATIME>
<MTIME>1402899229981</MTIME>
<ATIME>1402899229981</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>26</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>28</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -383,8 +383,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158722986</MTIME>
<ATIME>1394158722895</ATIME>
<MTIME>1402899230219</MTIME>
<ATIME>1402899229981</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@ -404,7 +404,7 @@
<GENSTAMP>1003</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@ -418,18 +418,18 @@
<INODEID>16390</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158722989</MTIME>
<ATIME>1394158722989</ATIME>
<MTIME>1402899230235</MTIME>
<ATIME>1402899230235</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>39</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>41</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -534,8 +534,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158723010</MTIME>
<ATIME>1394158722989</ATIME>
<MTIME>1402899230307</MTIME>
<ATIME>1402899230235</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@ -555,7 +555,7 @@
<GENSTAMP>1006</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@ -569,18 +569,18 @@
<INODEID>16391</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158723012</MTIME>
<ATIME>1394158723012</ATIME>
<MTIME>1402899230320</MTIME>
<ATIME>1402899230320</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>51</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>53</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -685,8 +685,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158723035</MTIME>
<ATIME>1394158723012</ATIME>
<MTIME>1402899230383</MTIME>
<ATIME>1402899230320</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@ -706,7 +706,7 @@
<GENSTAMP>1009</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@ -718,13 +718,13 @@
<TXID>56</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
<TIMESTAMP>1394158723039</TIMESTAMP>
<TIMESTAMP>1402899230394</TIMESTAMP>
<SOURCES>
<SOURCE1>/file_concat_0</SOURCE1>
<SOURCE2>/file_concat_1</SOURCE2>
</SOURCES>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>62</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>64</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -735,15 +735,15 @@
<INODEID>16392</INODEID>
<PATH>/file_symlink</PATH>
<VALUE>/file_concat_target</VALUE>
<MTIME>1394158723044</MTIME>
<ATIME>1394158723044</ATIME>
<MTIME>1402899230406</MTIME>
<ATIME>1402899230406</ATIME>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>511</MODE>
</PERMISSION_STATUS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>63</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>65</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -754,18 +754,18 @@
<INODEID>16393</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158723047</MTIME>
<ATIME>1394158723047</ATIME>
<MTIME>1402899230413</MTIME>
<ATIME>1402899230413</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>64</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>66</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -821,7 +821,7 @@
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
<TXID>64</TXID>
<LEASEHOLDER>DFSClient_NONMAPREDUCE_221786725_1</LEASEHOLDER>
<LEASEHOLDER>DFSClient_NONMAPREDUCE_1233039831_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
<NEWHOLDER>HDFS_NameNode</NEWHOLDER>
</DATA>
@ -834,8 +834,8 @@
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
<MTIME>1394158725708</MTIME>
<ATIME>1394158723047</ATIME>
<MTIME>1402899232526</MTIME>
<ATIME>1402899230413</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@ -845,7 +845,7 @@
<GENSTAMP>1011</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
<USERNAME>jing</USERNAME>
<USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@ -856,13 +856,13 @@
<DATA>
<TXID>66</TXID>
<POOLNAME>pool1</POOLNAME>
<OWNERNAME>jing</OWNERNAME>
<GROUPNAME>staff</GROUPNAME>
<OWNERNAME>andrew</OWNERNAME>
<GROUPNAME>andrew</GROUPNAME>
<MODE>493</MODE>
<LIMIT>9223372036854775807</LIMIT>
<MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>71</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>73</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -871,8 +871,8 @@
<TXID>67</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>72</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>74</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -883,9 +883,9 @@
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
<POOL>pool1</POOL>
<EXPIRATION>2305844403372420029</EXPIRATION>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>73</RPC_CALLID>
<EXPIRATION>2305844412112927450</EXPIRATION>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>75</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -894,8 +894,8 @@
<TXID>69</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>74</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>76</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -903,8 +903,8 @@
<DATA>
<TXID>70</TXID>
<ID>1</ID>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>75</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>77</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -912,8 +912,8 @@
<DATA>
<TXID>71</TXID>
<POOLNAME>pool1</POOLNAME>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CALLID>76</RPC_CALLID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>78</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@ -921,51 +921,91 @@
<DATA>
<TXID>72</TXID>
<SRC>/file_concat_target</SRC>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
<TXID>73</TXID>
<STARTTIME>1394158726098</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
<TXID>74</TXID>
<FINALIZETIME>1394158726098</FINALIZETIME>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
<TYPE>USER</TYPE>
<PERM>rw-</PERM>
</ENTRY>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
<TYPE>USER</TYPE>
<NAME>user</NAME>
<PERM>rw-</PERM>
</ENTRY>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
<TYPE>GROUP</TYPE>
<PERM>-w-</PERM>
</ENTRY>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
<TYPE>MASK</TYPE>
<PERM>rw-</PERM>
</ENTRY>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
<TYPE>OTHER</TYPE>
<PERM>---</PERM>
</ENTRY>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
<TXID>75</TXID>
<TXID>73</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a1</NAME>
<VALUE>0x313233</VALUE>
</XATTR>
<RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>80</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
<TXID>74</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
<VALUE>0x373839</VALUE>
</XATTR>
<RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>81</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_XATTR</OPCODE>
<DATA>
<TXID>76</TXID>
<TXID>75</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a1</NAME>
<NAME>a2</NAME>
</XATTR>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
<TXID>76</TXID>
<STARTTIME>1402899233646</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
<TXID>77</TXID>
<FINALIZETIME>1402899233647</FINALIZETIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_END_LOG_SEGMENT</OPCODE>
<DATA>
<TXID>77</TXID>
<TXID>78</TXID>
</DATA>
</RECORD>
</EDITS>