HDFS-5286. Flatten INodeDirectory hierarchy: Replace INodeDirectoryWithQuota with DirectoryWithQuotaFeature.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1545768 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2013-11-26 18:33:22 +00:00
parent 4a1acfc96f
commit 82ff2d3f2e
17 changed files with 324 additions and 231 deletions

View File

@ -215,6 +215,9 @@ Trunk (Unreleased)
HDFS-5538. URLConnectionFactory should pick up the SSL related configuration HDFS-5538. URLConnectionFactory should pick up the SSL related configuration
by default. (Haohui Mai via jing9) by default. (Haohui Mai via jing9)
HDFS-5286. Flatten INodeDirectory hierarchy: Replace INodeDirectoryWithQuota
with DirectoryWithQuotaFeature. (szetszwo)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe)

View File

@ -17,121 +17,76 @@
*/ */
package org.apache.hadoop.hdfs.server.namenode; package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import com.google.common.annotations.VisibleForTesting;
/** /**
* Directory INode class that has a quota restriction * Quota feature for {@link INodeDirectory}.
*/ */
public class INodeDirectoryWithQuota extends INodeDirectory { public final class DirectoryWithQuotaFeature extends INodeDirectory.Feature {
public static final long DEFAULT_NAMESPACE_QUOTA = Long.MAX_VALUE;
public static final long DEFAULT_DISKSPACE_QUOTA = HdfsConstants.QUOTA_RESET;
/** Name space quota */ /** Name space quota */
private long nsQuota = Long.MAX_VALUE; private long nsQuota = DEFAULT_NAMESPACE_QUOTA;
/** Name space count */ /** Name space count */
private long namespace = 1L; private long namespace = 1L;
/** Disk space quota */ /** Disk space quota */
private long dsQuota = HdfsConstants.QUOTA_RESET; private long dsQuota = DEFAULT_DISKSPACE_QUOTA;
/** Disk space count */ /** Disk space count */
private long diskspace = 0L; private long diskspace = 0L;
/** Convert an existing directory inode to one with the given quota DirectoryWithQuotaFeature(long nsQuota, long dsQuota) {
*
* @param nsQuota Namespace quota to be assigned to this inode
* @param dsQuota Diskspace quota to be assigned to this indoe
* @param other The other inode from which all other properties are copied
*/
INodeDirectoryWithQuota(INodeDirectory other, boolean adopt,
long nsQuota, long dsQuota) {
super(other, adopt);
final Quota.Counts counts = other.computeQuotaUsage();
this.namespace = counts.get(Quota.NAMESPACE);
this.diskspace = counts.get(Quota.DISKSPACE);
this.nsQuota = nsQuota; this.nsQuota = nsQuota;
this.dsQuota = dsQuota; this.dsQuota = dsQuota;
} }
public INodeDirectoryWithQuota(INodeDirectory other, boolean adopt, /** @return the quota set or -1 if it is not set. */
Quota.Counts quota) { Quota.Counts getQuota() {
this(other, adopt, quota.get(Quota.NAMESPACE), quota.get(Quota.DISKSPACE));
}
/** constructor with no quota verification */
INodeDirectoryWithQuota(long id, byte[] name, PermissionStatus permissions,
long modificationTime, long nsQuota, long dsQuota) {
super(id, name, permissions, modificationTime);
this.nsQuota = nsQuota;
this.dsQuota = dsQuota;
}
/** constructor with no quota verification */
INodeDirectoryWithQuota(long id, byte[] name, PermissionStatus permissions) {
super(id, name, permissions, 0L);
}
@Override
public Quota.Counts getQuotaCounts() {
return Quota.Counts.newInstance(nsQuota, dsQuota); return Quota.Counts.newInstance(nsQuota, dsQuota);
} }
/** Set this directory's quota /** Set this directory's quota
* *
* @param nsQuota Namespace quota to be set * @param nsQuota Namespace quota to be set
* @param dsQuota diskspace quota to be set * @param dsQuota Diskspace quota to be set
*/ */
public void setQuota(long nsQuota, long dsQuota) { void setQuota(long nsQuota, long dsQuota) {
this.nsQuota = nsQuota; this.nsQuota = nsQuota;
this.dsQuota = dsQuota; this.dsQuota = dsQuota;
} }
@Override Quota.Counts addNamespaceDiskspace(Quota.Counts counts) {
public Quota.Counts computeQuotaUsage(Quota.Counts counts, boolean useCache, counts.add(Quota.NAMESPACE, namespace);
int lastSnapshotId) { counts.add(Quota.DISKSPACE, diskspace);
if (useCache && isQuotaSet()) {
// use cache value
counts.add(Quota.NAMESPACE, namespace);
counts.add(Quota.DISKSPACE, diskspace);
} else {
super.computeQuotaUsage(counts, false, lastSnapshotId);
}
return counts; return counts;
} }
@Override ContentSummaryComputationContext computeContentSummary(final INodeDirectory dir,
public ContentSummaryComputationContext computeContentSummary(
final ContentSummaryComputationContext summary) { final ContentSummaryComputationContext summary) {
final long original = summary.getCounts().get(Content.DISKSPACE); final long original = summary.getCounts().get(Content.DISKSPACE);
long oldYieldCount = summary.getYieldCount(); long oldYieldCount = summary.getYieldCount();
super.computeContentSummary(summary); dir.computeDirectoryContentSummary(summary);
// Check only when the content has not changed in the middle. // Check only when the content has not changed in the middle.
if (oldYieldCount == summary.getYieldCount()) { if (oldYieldCount == summary.getYieldCount()) {
checkDiskspace(summary.getCounts().get(Content.DISKSPACE) - original); checkDiskspace(dir, summary.getCounts().get(Content.DISKSPACE) - original);
} }
return summary; return summary;
} }
private void checkDiskspace(final long computed) { private void checkDiskspace(final INodeDirectory dir, final long computed) {
if (-1 != getQuotaCounts().get(Quota.DISKSPACE) && diskspace != computed) { if (-1 != getQuota().get(Quota.DISKSPACE) && diskspace != computed) {
NameNode.LOG.error("BUG: Inconsistent diskspace for directory " NameNode.LOG.error("BUG: Inconsistent diskspace for directory "
+ getFullPathName() + ". Cached = " + diskspace + dir.getFullPathName() + ". Cached = " + diskspace
+ " != Computed = " + computed); + " != Computed = " + computed);
} }
} }
/** Get the number of names in the subtree rooted at this directory void addSpaceConsumed(final INodeDirectory dir, final long nsDelta,
* @return the size of the subtree rooted at this directory final long dsDelta, boolean verify) throws QuotaExceededException {
*/ if (dir.isQuotaSet()) {
long numItemsInTree() {
return namespace;
}
@Override
public final void addSpaceConsumed(final long nsDelta, final long dsDelta,
boolean verify) throws QuotaExceededException {
if (isQuotaSet()) {
// The following steps are important: // The following steps are important:
// check quotas in this inode and all ancestors before changing counts // check quotas in this inode and all ancestors before changing counts
// so that no change is made if there is any quota violation. // so that no change is made if there is any quota violation.
@ -141,11 +96,11 @@ public final void addSpaceConsumed(final long nsDelta, final long dsDelta,
verifyQuota(nsDelta, dsDelta); verifyQuota(nsDelta, dsDelta);
} }
// (2) verify quota and then add count in ancestors // (2) verify quota and then add count in ancestors
super.addSpaceConsumed(nsDelta, dsDelta, verify); dir.addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
// (3) add count in this inode // (3) add count in this inode
addSpaceConsumed2Cache(nsDelta, dsDelta); addSpaceConsumed2Cache(nsDelta, dsDelta);
} else { } else {
super.addSpaceConsumed(nsDelta, dsDelta, verify); dir.addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
} }
} }
@ -154,7 +109,7 @@ public final void addSpaceConsumed(final long nsDelta, final long dsDelta,
* @param nsDelta the change of the tree size * @param nsDelta the change of the tree size
* @param dsDelta change to disk space occupied * @param dsDelta change to disk space occupied
*/ */
protected void addSpaceConsumed2Cache(long nsDelta, long dsDelta) { public void addSpaceConsumed2Cache(long nsDelta, long dsDelta) {
namespace += nsDelta; namespace += nsDelta;
diskspace += dsDelta; diskspace += dsDelta;
} }
@ -172,41 +127,42 @@ void setSpaceConsumed(long namespace, long diskspace) {
this.diskspace = diskspace; this.diskspace = diskspace;
} }
/** @return the namespace and diskspace consumed. */
public Quota.Counts getSpaceConsumed() {
return Quota.Counts.newInstance(namespace, diskspace);
}
/** Verify if the namespace quota is violated after applying delta. */ /** Verify if the namespace quota is violated after applying delta. */
void verifyNamespaceQuota(long delta) throws NSQuotaExceededException { private void verifyNamespaceQuota(long delta) throws NSQuotaExceededException {
if (Quota.isViolated(nsQuota, namespace, delta)) { if (Quota.isViolated(nsQuota, namespace, delta)) {
throw new NSQuotaExceededException(nsQuota, namespace + delta); throw new NSQuotaExceededException(nsQuota, namespace + delta);
} }
} }
/** Verify if the diskspace quota is violated after applying delta. */
/** Verify if the namespace count disk space satisfies the quota restriction private void verifyDiskspaceQuota(long delta) throws DSQuotaExceededException {
* @throws QuotaExceededException if the given quota is less than the count if (Quota.isViolated(dsQuota, diskspace, delta)) {
*/ throw new DSQuotaExceededException(dsQuota, diskspace + delta);
void verifyQuota(long nsDelta, long dsDelta) throws QuotaExceededException {
verifyNamespaceQuota(nsDelta);
if (Quota.isViolated(dsQuota, diskspace, dsDelta)) {
throw new DSQuotaExceededException(dsQuota, diskspace + dsDelta);
} }
} }
String namespaceString() { /**
* @throws QuotaExceededException if namespace or diskspace quotas is
* violated after applying the deltas.
*/
void verifyQuota(long nsDelta, long dsDelta) throws QuotaExceededException {
verifyNamespaceQuota(nsDelta);
verifyDiskspaceQuota(dsDelta);
}
private String namespaceString() {
return "namespace: " + (nsQuota < 0? "-": namespace + "/" + nsQuota); return "namespace: " + (nsQuota < 0? "-": namespace + "/" + nsQuota);
} }
String diskspaceString() { private String diskspaceString() {
return "diskspace: " + (dsQuota < 0? "-": diskspace + "/" + dsQuota); return "diskspace: " + (dsQuota < 0? "-": diskspace + "/" + dsQuota);
} }
String quotaString() {
return ", Quota[" + namespaceString() + ", " + diskspaceString() + "]";
}
@VisibleForTesting @Override
public long getNamespace() { public String toString() {
return this.namespace; return "Quota[" + namespaceString() + ", " + diskspaceString() + "]";
}
@VisibleForTesting
public long getDiskspace() {
return this.diskspace;
} }
} }

View File

@ -86,11 +86,15 @@
* *
*************************************************/ *************************************************/
public class FSDirectory implements Closeable { public class FSDirectory implements Closeable {
private static INodeDirectoryWithQuota createRoot(FSNamesystem namesystem) { private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
final INodeDirectoryWithQuota r = new INodeDirectoryWithQuota( final INodeDirectory r = new INodeDirectory(
INodeId.ROOT_INODE_ID, INodeId.ROOT_INODE_ID,
INodeDirectory.ROOT_NAME, INodeDirectory.ROOT_NAME,
namesystem.createFsOwnerPermissions(new FsPermission((short) 0755))); namesystem.createFsOwnerPermissions(new FsPermission((short) 0755)),
0L);
r.addDirectoryWithQuotaFeature(
DirectoryWithQuotaFeature.DEFAULT_NAMESPACE_QUOTA,
DirectoryWithQuotaFeature.DEFAULT_DISKSPACE_QUOTA);
final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(r); final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(r);
s.setSnapshotQuota(0); s.setSnapshotQuota(0);
return s; return s;
@ -106,7 +110,7 @@ private static INodeDirectoryWithQuota createRoot(FSNamesystem namesystem) {
public final static String DOT_INODES_STRING = ".inodes"; public final static String DOT_INODES_STRING = ".inodes";
public final static byte[] DOT_INODES = public final static byte[] DOT_INODES =
DFSUtil.string2Bytes(DOT_INODES_STRING); DFSUtil.string2Bytes(DOT_INODES_STRING);
INodeDirectoryWithQuota rootDir; INodeDirectory rootDir;
FSImage fsImage; FSImage fsImage;
private final FSNamesystem namesystem; private final FSNamesystem namesystem;
private volatile boolean ready = false; private volatile boolean ready = false;
@ -201,7 +205,7 @@ private BlockManager getBlockManager() {
} }
/** @return the root directory inode. */ /** @return the root directory inode. */
public INodeDirectoryWithQuota getRoot() { public INodeDirectory getRoot() {
return rootDir; return rootDir;
} }
@ -1799,9 +1803,8 @@ private static void unprotectedUpdateCount(INodesInPath inodesInPath,
final INode[] inodes = inodesInPath.getINodes(); final INode[] inodes = inodesInPath.getINodes();
for(int i=0; i < numOfINodes; i++) { for(int i=0; i < numOfINodes; i++) {
if (inodes[i].isQuotaSet()) { // a directory with quota if (inodes[i].isQuotaSet()) { // a directory with quota
INodeDirectoryWithQuota node = (INodeDirectoryWithQuota) inodes[i] inodes[i].asDirectory().getDirectoryWithQuotaFeature()
.asDirectory(); .addSpaceConsumed2Cache(nsDelta, dsDelta);
node.addSpaceConsumed2Cache(nsDelta, dsDelta);
} }
} }
} }
@ -2034,10 +2037,11 @@ private static void verifyQuota(INode[] inodes, int pos, long nsDelta,
// Stop checking for quota when common ancestor is reached // Stop checking for quota when common ancestor is reached
return; return;
} }
if (inodes[i].isQuotaSet()) { // a directory with quota final DirectoryWithQuotaFeature q
= inodes[i].asDirectory().getDirectoryWithQuotaFeature();
if (q != null) { // a directory with quota
try { try {
((INodeDirectoryWithQuota) inodes[i].asDirectory()).verifyQuota( q.verifyQuota(nsDelta, dsDelta);
nsDelta, dsDelta);
} catch (QuotaExceededException e) { } catch (QuotaExceededException e) {
e.setPathName(getFullPathName(inodes, i)); e.setPathName(getFullPathName(inodes, i));
throw e; throw e;
@ -2384,35 +2388,14 @@ INodeDirectory unprotectedSetQuota(String src, long nsQuota, long dsQuota)
if (dsQuota == HdfsConstants.QUOTA_DONT_SET) { if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
dsQuota = oldDsQuota; dsQuota = oldDsQuota;
} }
if (oldNsQuota == nsQuota && oldDsQuota == dsQuota) {
return null;
}
final Snapshot latest = iip.getLatestSnapshot(); final Snapshot latest = iip.getLatestSnapshot();
if (dirNode instanceof INodeDirectoryWithQuota) { dirNode = dirNode.recordModification(latest, inodeMap);
INodeDirectoryWithQuota quotaNode = (INodeDirectoryWithQuota) dirNode; dirNode.setQuota(nsQuota, dsQuota);
Quota.Counts counts = null; return dirNode;
if (!quotaNode.isQuotaSet()) {
// dirNode must be an INodeDirectoryWithSnapshot whose quota has not
// been set yet
counts = quotaNode.computeQuotaUsage();
}
// a directory with quota; so set the quota to the new value
quotaNode.setQuota(nsQuota, dsQuota);
if (quotaNode.isQuotaSet() && counts != null) {
quotaNode.setSpaceConsumed(counts.get(Quota.NAMESPACE),
counts.get(Quota.DISKSPACE));
} else if (!quotaNode.isQuotaSet() && latest == null) {
// do not replace the node if the node is a snapshottable directory
// without snapshots
if (!(quotaNode instanceof INodeDirectoryWithSnapshot)) {
// will not come here for root because root is snapshottable and
// root's nsQuota is always set
return quotaNode.replaceSelf4INodeDirectory(inodeMap);
}
}
} else {
// a non-quota directory; so replace it with a directory with quota
return dirNode.replaceSelf4Quota(latest, nsQuota, dsQuota, inodeMap);
}
return (oldNsQuota != nsQuota || oldDsQuota != dsQuota) ? dirNode : null;
} }
} }
@ -2441,7 +2424,8 @@ void setQuota(String src, long nsQuota, long dsQuota)
long totalInodes() { long totalInodes() {
readLock(); readLock();
try { try {
return rootDir.numItemsInTree(); return rootDir.getDirectoryWithQuotaFeature().getSpaceConsumed()
.get(Quota.NAMESPACE);
} finally { } finally {
readUnlock(); readUnlock();
} }

View File

@ -755,7 +755,7 @@ public long loadEdits(Iterable<EditLogInputStream> editStreams,
* This is an update of existing state of the filesystem and does not * This is an update of existing state of the filesystem and does not
* throw QuotaExceededException. * throw QuotaExceededException.
*/ */
static void updateCountForQuota(INodeDirectoryWithQuota root) { static void updateCountForQuota(INodeDirectory root) {
updateCountForQuotaRecursively(root, Quota.Counts.newInstance()); updateCountForQuotaRecursively(root, Quota.Counts.newInstance());
} }
@ -795,7 +795,7 @@ private static void updateCountForQuotaRecursively(INodeDirectory dir,
+ " quota = " + dsQuota + " < consumed = " + diskspace); + " quota = " + dsQuota + " < consumed = " + diskspace);
} }
((INodeDirectoryWithQuota)dir).setSpaceConsumed(namespace, diskspace); dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, diskspace);
} }
} }

View File

@ -375,7 +375,7 @@ private void updateRootAttr(INodeWithAdditionalFields root) {
final long dsQuota = q.get(Quota.DISKSPACE); final long dsQuota = q.get(Quota.DISKSPACE);
FSDirectory fsDir = namesystem.dir; FSDirectory fsDir = namesystem.dir;
if (nsQuota != -1 || dsQuota != -1) { if (nsQuota != -1 || dsQuota != -1) {
fsDir.rootDir.setQuota(nsQuota, dsQuota); fsDir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
} }
fsDir.rootDir.cloneModificationTime(root); fsDir.rootDir.cloneModificationTime(root);
fsDir.rootDir.clonePermissionStatus(root); fsDir.rootDir.clonePermissionStatus(root);
@ -729,10 +729,11 @@ INode loadINode(final byte[] localName, boolean isSnapshotINode,
if (counter != null) { if (counter != null) {
counter.increment(); counter.increment();
} }
final INodeDirectory dir = nsQuota >= 0 || dsQuota >= 0? final INodeDirectory dir = new INodeDirectory(inodeId, localName,
new INodeDirectoryWithQuota(inodeId, localName, permissions, permissions, modificationTime);
modificationTime, nsQuota, dsQuota) if (nsQuota >= 0 || dsQuota >= 0) {
: new INodeDirectory(inodeId, localName, permissions, modificationTime); dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota);
}
return snapshottable ? new INodeDirectorySnapshottable(dir) return snapshottable ? new INodeDirectorySnapshottable(dir)
: withSnapshot ? new INodeDirectoryWithSnapshot(dir) : withSnapshot ? new INodeDirectoryWithSnapshot(dir)
: dir; : dir;
@ -972,13 +973,14 @@ void save(File newFile, FSImageCompression compression) throws IOException {
checkNotSaved(); checkNotSaved();
final FSNamesystem sourceNamesystem = context.getSourceNamesystem(); final FSNamesystem sourceNamesystem = context.getSourceNamesystem();
FSDirectory fsDir = sourceNamesystem.dir; final INodeDirectory rootDir = sourceNamesystem.dir.rootDir;
final long numINodes = rootDir.getDirectoryWithQuotaFeature()
.getSpaceConsumed().get(Quota.NAMESPACE);
String sdPath = newFile.getParentFile().getParentFile().getAbsolutePath(); String sdPath = newFile.getParentFile().getParentFile().getAbsolutePath();
Step step = new Step(StepType.INODES, sdPath); Step step = new Step(StepType.INODES, sdPath);
StartupProgress prog = NameNode.getStartupProgress(); StartupProgress prog = NameNode.getStartupProgress();
prog.beginStep(Phase.SAVING_CHECKPOINT, step); prog.beginStep(Phase.SAVING_CHECKPOINT, step);
prog.setTotal(Phase.SAVING_CHECKPOINT, step, prog.setTotal(Phase.SAVING_CHECKPOINT, step, numINodes);
fsDir.rootDir.numItemsInTree());
Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step); Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
long startTime = now(); long startTime = now();
// //
@ -997,7 +999,7 @@ void save(File newFile, FSImageCompression compression) throws IOException {
// fairness-related deadlock. See the comments on HDFS-2223. // fairness-related deadlock. See the comments on HDFS-2223.
out.writeInt(sourceNamesystem.unprotectedGetNamespaceInfo() out.writeInt(sourceNamesystem.unprotectedGetNamespaceInfo()
.getNamespaceID()); .getNamespaceID());
out.writeLong(fsDir.rootDir.numItemsInTree()); out.writeLong(numINodes);
out.writeLong(sourceNamesystem.getGenerationStampV1()); out.writeLong(sourceNamesystem.getGenerationStampV1());
out.writeLong(sourceNamesystem.getGenerationStampV2()); out.writeLong(sourceNamesystem.getGenerationStampV2());
out.writeLong(sourceNamesystem.getGenerationStampAtblockIdSwitch()); out.writeLong(sourceNamesystem.getGenerationStampAtblockIdSwitch());
@ -1014,14 +1016,13 @@ void save(File newFile, FSImageCompression compression) throws IOException {
" using " + compression); " using " + compression);
// save the root // save the root
saveINode2Image(fsDir.rootDir, out, false, referenceMap, counter); saveINode2Image(rootDir, out, false, referenceMap, counter);
// save the rest of the nodes // save the rest of the nodes
saveImage(fsDir.rootDir, out, true, false, counter); saveImage(rootDir, out, true, false, counter);
prog.endStep(Phase.SAVING_CHECKPOINT, step); prog.endStep(Phase.SAVING_CHECKPOINT, step);
// Now that the step is finished, set counter equal to total to adjust // Now that the step is finished, set counter equal to total to adjust
// for possible under-counting due to reference inodes. // for possible under-counting due to reference inodes.
prog.setCount(Phase.SAVING_CHECKPOINT, step, prog.setCount(Phase.SAVING_CHECKPOINT, step, numINodes);
fsDir.rootDir.numItemsInTree());
// save files under construction // save files under construction
// TODO: for HDFS-5428, since we cannot break the compatibility of // TODO: for HDFS-5428, since we cannot break the compatibility of
// fsimage, we store part of the under-construction files that are only // fsimage, we store part of the under-construction files that are only

View File

@ -406,6 +406,15 @@ public abstract ContentSummaryComputationContext computeContentSummary(
*/ */
public void addSpaceConsumed(long nsDelta, long dsDelta, boolean verify) public void addSpaceConsumed(long nsDelta, long dsDelta, boolean verify)
throws QuotaExceededException { throws QuotaExceededException {
addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
}
/**
* Check and add namespace/diskspace consumed to itself and the ancestors.
* @throws QuotaExceededException if quote is violated.
*/
void addSpaceConsumed2Parent(long nsDelta, long dsDelta, boolean verify)
throws QuotaExceededException {
if (parent != null) { if (parent != null) {
parent.addSpaceConsumed(nsDelta, dsDelta, verify); parent.addSpaceConsumed(nsDelta, dsDelta, verify);
} }
@ -744,4 +753,51 @@ public void clear() {
toDeleteList.clear(); toDeleteList.clear();
} }
} }
/** INode feature such as {@link FileUnderConstructionFeature}
* and {@link DirectoryWithQuotaFeature}.
*/
interface Feature<F extends Feature<F>> {
/** @return the next feature. */
public F getNextFeature();
/** Set the next feature. */
public void setNextFeature(F next);
/** Utility methods such as addFeature and removeFeature. */
static class Util {
/**
* Add a feature to the linked list.
* @return the new head.
*/
static <F extends Feature<F>> F addFeature(F feature, F head) {
feature.setNextFeature(head);
return feature;
}
/**
* Remove a feature from the linked list.
* @return the new head.
*/
static <F extends Feature<F>> F removeFeature(F feature, F head) {
if (feature == head) {
final F newHead = head.getNextFeature();
head.setNextFeature(null);
return newHead;
} else if (head != null) {
F prev = head;
F curr = head.getNextFeature();
for (; curr != null && curr != feature;
prev = curr, curr = curr.getNextFeature())
;
if (curr != null) {
prev.setNextFeature(curr.getNextFeature());
curr.setNextFeature(null);
return head;
}
}
throw new IllegalStateException("Feature " + feature + " not found.");
}
}
}
} }

View File

@ -46,6 +46,21 @@
*/ */
public class INodeDirectory extends INodeWithAdditionalFields public class INodeDirectory extends INodeWithAdditionalFields
implements INodeDirectoryAttributes { implements INodeDirectoryAttributes {
/** Directory related features such as quota and snapshots. */
public static abstract class Feature implements INode.Feature<Feature> {
private Feature nextFeature;
@Override
public Feature getNextFeature() {
return nextFeature;
}
@Override
public void setNextFeature(Feature next) {
this.nextFeature = next;
}
}
/** Cast INode to INodeDirectory. */ /** Cast INode to INodeDirectory. */
public static INodeDirectory valueOf(INode inode, Object path public static INodeDirectory valueOf(INode inode, Object path
) throws FileNotFoundException, PathIsNotDirectoryException { ) throws FileNotFoundException, PathIsNotDirectoryException {
@ -64,6 +79,9 @@ public static INodeDirectory valueOf(INode inode, Object path
private List<INode> children = null; private List<INode> children = null;
/** A linked list of {@link Feature}s. */
private Feature headFeature = null;
/** constructor */ /** constructor */
public INodeDirectory(long id, byte[] name, PermissionStatus permissions, public INodeDirectory(long id, byte[] name, PermissionStatus permissions,
long mtime) { long mtime) {
@ -76,7 +94,7 @@ public INodeDirectory(long id, byte[] name, PermissionStatus permissions,
* @param adopt Indicate whether or not need to set the parent field of child * @param adopt Indicate whether or not need to set the parent field of child
* INodes to the new node * INodes to the new node
*/ */
public INodeDirectory(INodeDirectory other, boolean adopt) { public INodeDirectory(INodeDirectory other, boolean adopt, boolean copyFeatures) {
super(other); super(other);
this.children = other.children; this.children = other.children;
if (adopt && this.children != null) { if (adopt && this.children != null) {
@ -84,6 +102,9 @@ public INodeDirectory(INodeDirectory other, boolean adopt) {
child.setParent(this); child.setParent(this);
} }
} }
if (copyFeatures) {
this.headFeature = other.headFeature;
}
} }
/** @return true unconditionally. */ /** @return true unconditionally. */
@ -103,6 +124,73 @@ public boolean isSnapshottable() {
return false; return false;
} }
void setQuota(long nsQuota, long dsQuota) {
DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature();
if (quota != null) {
// already has quota; so set the quota to the new values
quota.setQuota(nsQuota, dsQuota);
if (!isQuotaSet() && !isRoot()) {
removeFeature(quota);
}
} else {
final Quota.Counts c = computeQuotaUsage();
quota = addDirectoryWithQuotaFeature(nsQuota, dsQuota);
quota.setSpaceConsumed(c.get(Quota.NAMESPACE), c.get(Quota.DISKSPACE));
}
}
@Override
public Quota.Counts getQuotaCounts() {
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
return q != null? q.getQuota(): super.getQuotaCounts();
}
@Override
public void addSpaceConsumed(long nsDelta, long dsDelta, boolean verify)
throws QuotaExceededException {
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null) {
q.addSpaceConsumed(this, nsDelta, dsDelta, verify);
} else {
addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
}
}
/**
* If the directory contains a {@link DirectoryWithQuotaFeature}, return it;
* otherwise, return null.
*/
public final DirectoryWithQuotaFeature getDirectoryWithQuotaFeature() {
for(Feature f = headFeature; f != null; f = f.nextFeature) {
if (f instanceof DirectoryWithQuotaFeature) {
return (DirectoryWithQuotaFeature)f;
}
}
return null;
}
/** Is this directory with quota? */
final boolean isWithQuota() {
return getDirectoryWithQuotaFeature() != null;
}
DirectoryWithQuotaFeature addDirectoryWithQuotaFeature(
long nsQuota, long dsQuota) {
Preconditions.checkState(!isWithQuota(), "Directory is already with quota");
final DirectoryWithQuotaFeature quota = new DirectoryWithQuotaFeature(
nsQuota, dsQuota);
addFeature(quota);
return quota;
}
private void addFeature(Feature f) {
headFeature = INode.Feature.Util.addFeature(f, headFeature);
}
private void removeFeature(Feature f) {
headFeature = INode.Feature.Util.removeFeature(f, headFeature);
}
private int searchChildren(byte[] name) { private int searchChildren(byte[] name) {
return children == null? -1: Collections.binarySearch(children, name); return children == null? -1: Collections.binarySearch(children, name);
} }
@ -142,27 +230,6 @@ protected final boolean removeChild(final INode child) {
return true; return true;
} }
/**
* Replace itself with {@link INodeDirectoryWithQuota} or
* {@link INodeDirectoryWithSnapshot} depending on the latest snapshot.
*/
INodeDirectoryWithQuota replaceSelf4Quota(final Snapshot latest,
final long nsQuota, final long dsQuota, final INodeMap inodeMap)
throws QuotaExceededException {
Preconditions.checkState(!(this instanceof INodeDirectoryWithQuota),
"this is already an INodeDirectoryWithQuota, this=%s", this);
if (!this.isInLatestSnapshot(latest)) {
final INodeDirectoryWithQuota q = new INodeDirectoryWithQuota(
this, true, nsQuota, dsQuota);
replaceSelf(q, inodeMap);
return q;
} else {
final INodeDirectoryWithSnapshot s = new INodeDirectoryWithSnapshot(this);
s.setQuota(nsQuota, dsQuota);
return replaceSelf(s, inodeMap).saveSelf2Snapshot(latest, this);
}
}
/** Replace itself with an {@link INodeDirectorySnapshottable}. */ /** Replace itself with an {@link INodeDirectorySnapshottable}. */
public INodeDirectorySnapshottable replaceSelf4INodeDirectorySnapshottable( public INodeDirectorySnapshottable replaceSelf4INodeDirectorySnapshottable(
Snapshot latest, final INodeMap inodeMap) throws QuotaExceededException { Snapshot latest, final INodeMap inodeMap) throws QuotaExceededException {
@ -183,7 +250,7 @@ public INodeDirectoryWithSnapshot replaceSelf4INodeDirectoryWithSnapshot(
public INodeDirectory replaceSelf4INodeDirectory(final INodeMap inodeMap) { public INodeDirectory replaceSelf4INodeDirectory(final INodeMap inodeMap) {
Preconditions.checkState(getClass() != INodeDirectory.class, Preconditions.checkState(getClass() != INodeDirectory.class,
"the class is already INodeDirectory, this=%s", this); "the class is already INodeDirectory, this=%s", this);
return replaceSelf(new INodeDirectory(this, true), inodeMap); return replaceSelf(new INodeDirectory(this, true, true), inodeMap);
} }
/** Replace itself with the given directory. */ /** Replace itself with the given directory. */
@ -439,6 +506,21 @@ private void addChild(final INode node, final int insertionPoint) {
@Override @Override
public Quota.Counts computeQuotaUsage(Quota.Counts counts, boolean useCache, public Quota.Counts computeQuotaUsage(Quota.Counts counts, boolean useCache,
int lastSnapshotId) { int lastSnapshotId) {
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null) {
if (useCache && isQuotaSet()) {
q.addNamespaceDiskspace(counts);
} else {
computeDirectoryQuotaUsage(counts, false, lastSnapshotId);
}
return counts;
} else {
return computeDirectoryQuotaUsage(counts, useCache, lastSnapshotId);
}
}
Quota.Counts computeDirectoryQuotaUsage(Quota.Counts counts, boolean useCache,
int lastSnapshotId) {
if (children != null) { if (children != null) {
for (INode child : children) { for (INode child : children) {
child.computeQuotaUsage(counts, useCache, lastSnapshotId); child.computeQuotaUsage(counts, useCache, lastSnapshotId);
@ -456,6 +538,16 @@ public Quota.Counts computeQuotaUsage4CurrentDirectory(Quota.Counts counts) {
@Override @Override
public ContentSummaryComputationContext computeContentSummary( public ContentSummaryComputationContext computeContentSummary(
ContentSummaryComputationContext summary) { ContentSummaryComputationContext summary) {
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null) {
return q.computeContentSummary(this, summary);
} else {
return computeDirectoryContentSummary(summary);
}
}
ContentSummaryComputationContext computeDirectoryContentSummary(
ContentSummaryComputationContext summary) {
ReadOnlyList<INode> childrenList = getChildrenList(null); ReadOnlyList<INode> childrenList = getChildrenList(null);
// Explicit traversing is done to enable repositioning after relinquishing // Explicit traversing is done to enable repositioning after relinquishing
// and reacquiring locks. // and reacquiring locks.
@ -570,7 +662,7 @@ public Quota.Counts cleanSubtree(final Snapshot snapshot, Snapshot prior,
Quota.Counts counts = cleanSubtreeRecursively(snapshot, prior, Quota.Counts counts = cleanSubtreeRecursively(snapshot, prior,
collectedBlocks, removedINodes, null, countDiffChange); collectedBlocks, removedINodes, null, countDiffChange);
if (isQuotaSet()) { if (isQuotaSet()) {
((INodeDirectoryWithQuota) this).addSpaceConsumed2Cache( getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(
-counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE)); -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
} }
return counts; return counts;
@ -606,8 +698,9 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
final Snapshot snapshot) { final Snapshot snapshot) {
super.dumpTreeRecursively(out, prefix, snapshot); super.dumpTreeRecursively(out, prefix, snapshot);
out.print(", childrenSize=" + getChildrenList(snapshot).size()); out.print(", childrenSize=" + getChildrenList(snapshot).size());
if (this instanceof INodeDirectoryWithQuota) { final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
out.print(((INodeDirectoryWithQuota)this).quotaString()); if (q != null) {
out.print(", " + q);
} }
if (this instanceof Snapshot.Root) { if (this instanceof Snapshot.Root) {
out.print(", snapshotId=" + snapshot.getId()); out.print(", snapshotId=" + snapshot.getId());

View File

@ -50,13 +50,15 @@ public class INodeFile extends INodeWithAdditionalFields
* A feature contains specific information for a type of INodeFile. E.g., * A feature contains specific information for a type of INodeFile. E.g.,
* we can have separate features for Under-Construction and Snapshot. * we can have separate features for Under-Construction and Snapshot.
*/ */
public static abstract class Feature { public static abstract class Feature implements INode.Feature<Feature> {
private Feature nextFeature; private Feature nextFeature;
@Override
public Feature getNextFeature() { public Feature getNextFeature() {
return nextFeature; return nextFeature;
} }
@Override
public void setNextFeature(Feature next) { public void setNextFeature(Feature next) {
this.nextFeature = next; this.nextFeature = next;
} }
@ -160,26 +162,12 @@ public boolean isUnderConstruction() {
return getFileUnderConstructionFeature() != null; return getFileUnderConstructionFeature() != null;
} }
void addFeature(Feature f) { private void addFeature(Feature f) {
f.nextFeature = headFeature; headFeature = INode.Feature.Util.addFeature(f, headFeature);
headFeature = f;
} }
void removeFeature(Feature f) { private void removeFeature(Feature f) {
if (f == headFeature) { headFeature = INode.Feature.Util.removeFeature(f, headFeature);
headFeature = headFeature.nextFeature;
return;
} else if (headFeature != null) {
Feature prev = headFeature;
Feature curr = headFeature.nextFeature;
for (; curr != null && curr != f; prev = curr, curr = curr.nextFeature)
;
if (curr != null) {
prev.nextFeature = curr.nextFeature;
return;
}
}
throw new IllegalStateException("Feature " + f + " not found.");
} }
/** @return true unconditionally. */ /** @return true unconditionally. */
@ -197,7 +185,7 @@ public final INodeFile asFile() {
/* Start of Under-Construction Feature */ /* Start of Under-Construction Feature */
/** Convert this file to an {@link INodeFileUnderConstruction}. */ /** Convert this file to an {@link INodeFileUnderConstruction}. */
public INodeFile toUnderConstruction(String clientName, String clientMachine, INodeFile toUnderConstruction(String clientName, String clientMachine,
DatanodeDescriptor clientNode) { DatanodeDescriptor clientNode) {
Preconditions.checkState(!isUnderConstruction(), Preconditions.checkState(!isUnderConstruction(),
"file is already an INodeFileUnderConstruction"); "file is already an INodeFileUnderConstruction");

View File

@ -37,7 +37,6 @@
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryWithQuota;
import org.apache.hadoop.hdfs.server.namenode.INodeMap; import org.apache.hadoop.hdfs.server.namenode.INodeMap;
import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.Quota; import org.apache.hadoop.hdfs.server.namenode.Quota;
@ -55,7 +54,7 @@
* storing snapshot data. When there are modifications to the directory, the old * storing snapshot data. When there are modifications to the directory, the old
* data is stored in the latest snapshot, if there is any. * data is stored in the latest snapshot, if there is any.
*/ */
public class INodeDirectoryWithSnapshot extends INodeDirectoryWithQuota { public class INodeDirectoryWithSnapshot extends INodeDirectory {
/** /**
* The difference between the current state and a previous snapshot * The difference between the current state and a previous snapshot
* of the children list of an INodeDirectory. * of the children list of an INodeDirectory.
@ -486,7 +485,7 @@ public INodeDirectoryWithSnapshot(INodeDirectory that) {
INodeDirectoryWithSnapshot(INodeDirectory that, boolean adopt, INodeDirectoryWithSnapshot(INodeDirectory that, boolean adopt,
DirectoryDiffList diffs) { DirectoryDiffList diffs) {
super(that, adopt, that.getQuotaCounts()); super(that, adopt, true);
this.diffs = diffs != null? diffs: new DirectoryDiffList(); this.diffs = diffs != null? diffs: new DirectoryDiffList();
} }
@ -771,8 +770,8 @@ public Quota.Counts cleanSubtree(final Snapshot snapshot, Snapshot prior,
removedINodes, priorDeleted, countDiffChange)); removedINodes, priorDeleted, countDiffChange));
if (isQuotaSet()) { if (isQuotaSet()) {
this.addSpaceConsumed2Cache(-counts.get(Quota.NAMESPACE), getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(
-counts.get(Quota.DISKSPACE)); -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
} }
return counts; return counts;
} }

View File

@ -136,7 +136,7 @@ static Snapshot read(DataInput in, FSImageFormat.Loader loader)
/** The root directory of the snapshot. */ /** The root directory of the snapshot. */
static public class Root extends INodeDirectory { static public class Root extends INodeDirectory {
Root(INodeDirectory other) { Root(INodeDirectory other) {
super(other, false); super(other, false, false);
} }
@Override @Override

View File

@ -368,10 +368,7 @@ public void testNamespaceCommands() throws Exception {
// be identical. // be identical.
conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2); conf.setInt(DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, 2);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
final FileSystem fs = cluster.getFileSystem(); final DistributedFileSystem dfs = cluster.getFileSystem();
assertTrue("Not a HDFS: "+fs.getUri(),
fs instanceof DistributedFileSystem);
final DistributedFileSystem dfs = (DistributedFileSystem)fs;
try { try {
// 1: create directory /nqdir0/qdir1/qdir20/nqdir30 // 1: create directory /nqdir0/qdir1/qdir20/nqdir30

View File

@ -158,7 +158,7 @@ private void loadFSImageFromTempFile(File imageFile) throws IOException {
try { try {
loader.load(imageFile); loader.load(imageFile);
FSImage.updateCountForQuota( FSImage.updateCountForQuota(
(INodeDirectoryWithQuota)fsn.getFSDirectory().getINode("/")); INodeDirectory.valueOf(fsn.getFSDirectory().getINode("/"), "/"));
} finally { } finally {
fsn.getFSDirectory().writeUnlock(); fsn.getFSDirectory().writeUnlock();
fsn.writeUnlock(); fsn.writeUnlock();

View File

@ -49,7 +49,7 @@ public class TestFsLimits {
static PermissionStatus perms static PermissionStatus perms
= new PermissionStatus("admin", "admin", FsPermission.getDefault()); = new PermissionStatus("admin", "admin", FsPermission.getDefault());
static INodeDirectoryWithQuota rootInode; static INodeDirectory rootInode;
static private FSNamesystem getMockNamesystem() { static private FSNamesystem getMockNamesystem() {
FSNamesystem fsn = mock(FSNamesystem.class); FSNamesystem fsn = mock(FSNamesystem.class);
@ -75,8 +75,8 @@ public void setUp() throws IOException {
fileAsURI(new File(MiniDFSCluster.getBaseDirectory(), fileAsURI(new File(MiniDFSCluster.getBaseDirectory(),
"namenode")).toString()); "namenode")).toString());
rootInode = new INodeDirectoryWithQuota(getMockNamesystem() rootInode = new INodeDirectory(getMockNamesystem().allocateNewInodeId(),
.allocateNewInodeId(), INodeDirectory.ROOT_NAME, perms); INodeDirectory.ROOT_NAME, perms, 0L);
inodes = new INode[]{ rootInode, null }; inodes = new INode[]{ rootInode, null };
fs = null; fs = null;
fsIsReady = true; fsIsReady = true;

View File

@ -211,9 +211,9 @@ public void testGetFullPathNameAfterSetQuota() throws Exception {
// Call FSDirectory#unprotectedSetQuota which calls // Call FSDirectory#unprotectedSetQuota which calls
// INodeDirectory#replaceChild // INodeDirectory#replaceChild
dfs.setQuota(dir, Long.MAX_VALUE - 1, replication * fileLen * 10); dfs.setQuota(dir, Long.MAX_VALUE - 1, replication * fileLen * 10);
INode dirNode = fsdir.getINode(dir.toString()); INodeDirectory dirNode = getDir(fsdir, dir);
assertEquals(dir.toString(), dirNode.getFullPathName()); assertEquals(dir.toString(), dirNode.getFullPathName());
assertTrue(dirNode instanceof INodeDirectoryWithQuota); assertTrue(dirNode.isWithQuota());
final Path newDir = new Path("/newdir"); final Path newDir = new Path("/newdir");
final Path newFile = new Path(newDir, "file"); final Path newFile = new Path(newDir, "file");
@ -871,6 +871,12 @@ public void testInodePath() throws IOException {
} }
} }
private static INodeDirectory getDir(final FSDirectory fsdir, final Path dir)
throws IOException {
final String dirStr = dir.toString();
return INodeDirectory.valueOf(fsdir.getINode(dirStr), dirStr);
}
/** /**
* Test whether the inode in inodeMap has been replaced after regular inode * Test whether the inode in inodeMap has been replaced after regular inode
* replacement * replacement
@ -887,21 +893,20 @@ public void testInodeReplacement() throws Exception {
final Path dir = new Path("/dir"); final Path dir = new Path("/dir");
hdfs.mkdirs(dir); hdfs.mkdirs(dir);
INode dirNode = fsdir.getINode(dir.toString()); INodeDirectory dirNode = getDir(fsdir, dir);
INode dirNodeFromNode = fsdir.getInode(dirNode.getId()); INode dirNodeFromNode = fsdir.getInode(dirNode.getId());
assertSame(dirNode, dirNodeFromNode); assertSame(dirNode, dirNodeFromNode);
// set quota to dir, which leads to node replacement // set quota to dir, which leads to node replacement
hdfs.setQuota(dir, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1); hdfs.setQuota(dir, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
dirNode = fsdir.getINode(dir.toString()); dirNode = getDir(fsdir, dir);
assertTrue(dirNode instanceof INodeDirectoryWithQuota); assertTrue(dirNode.isWithQuota());
// the inode in inodeMap should also be replaced // the inode in inodeMap should also be replaced
dirNodeFromNode = fsdir.getInode(dirNode.getId()); dirNodeFromNode = fsdir.getInode(dirNode.getId());
assertSame(dirNode, dirNodeFromNode); assertSame(dirNode, dirNodeFromNode);
hdfs.setQuota(dir, -1, -1); hdfs.setQuota(dir, -1, -1);
dirNode = fsdir.getINode(dir.toString()); dirNode = getDir(fsdir, dir);
assertTrue(dirNode instanceof INodeDirectory);
// the inode in inodeMap should also be replaced // the inode in inodeMap should also be replaced
dirNodeFromNode = fsdir.getInode(dirNode.getId()); dirNodeFromNode = fsdir.getInode(dirNode.getId());
assertSame(dirNode, dirNodeFromNode); assertSame(dirNode, dirNodeFromNode);

View File

@ -1190,13 +1190,15 @@ public void testRenameDirAndDeleteSnapshot_2() throws Exception {
assertFalse(hdfs.exists(bar_s2)); assertFalse(hdfs.exists(bar_s2));
restartClusterAndCheckImage(true); restartClusterAndCheckImage(true);
// make sure the whole referred subtree has been destroyed // make sure the whole referred subtree has been destroyed
assertEquals(4, fsdir.getRoot().getNamespace()); Quota.Counts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(0, fsdir.getRoot().getDiskspace()); assertEquals(4, q.get(Quota.NAMESPACE));
assertEquals(0, q.get(Quota.DISKSPACE));
hdfs.deleteSnapshot(sdir1, "s1"); hdfs.deleteSnapshot(sdir1, "s1");
restartClusterAndCheckImage(true); restartClusterAndCheckImage(true);
assertEquals(3, fsdir.getRoot().getNamespace()); q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(0, fsdir.getRoot().getDiskspace()); assertEquals(3, q.get(Quota.NAMESPACE));
assertEquals(0, q.get(Quota.DISKSPACE));
} }
/** /**
@ -1938,10 +1940,12 @@ public void testRenameDirAndDeleteSnapshot_3() throws Exception {
// check // check
final INodeDirectorySnapshottable dir1Node = final INodeDirectorySnapshottable dir1Node =
(INodeDirectorySnapshottable) fsdir.getINode4Write(sdir1.toString()); (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir1.toString());
assertEquals(4, dir1Node.getNamespace()); Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(4, q1.get(Quota.NAMESPACE));
final INodeDirectorySnapshottable dir2Node = final INodeDirectorySnapshottable dir2Node =
(INodeDirectorySnapshottable) fsdir.getINode4Write(sdir2.toString()); (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir2.toString());
assertEquals(2, dir2Node.getNamespace()); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(2, q2.get(Quota.NAMESPACE));
final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
foo.getName()); foo.getName());
@ -2005,10 +2009,12 @@ public void testRenameDirAndDeleteSnapshot_4() throws Exception {
final INodeDirectorySnapshottable dir1Node = final INodeDirectorySnapshottable dir1Node =
(INodeDirectorySnapshottable) fsdir.getINode4Write(sdir1.toString()); (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir1.toString());
// sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3) // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3)
assertEquals(9, dir1Node.getNamespace()); Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(9, q1.get(Quota.NAMESPACE));
final INodeDirectorySnapshottable dir2Node = final INodeDirectorySnapshottable dir2Node =
(INodeDirectorySnapshottable) fsdir.getINode4Write(sdir2.toString()); (INodeDirectorySnapshottable) fsdir.getINode4Write(sdir2.toString());
assertEquals(2, dir2Node.getNamespace()); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(2, q2.get(Quota.NAMESPACE));
final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
foo.getName()); foo.getName());

View File

@ -46,7 +46,6 @@
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryWithQuota;
import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@ -157,15 +156,21 @@ public void testDeleteDirectoryWithSnapshot2() throws Exception {
hdfs.delete(dir, true); hdfs.delete(dir, true);
} }
private static INodeDirectory getDir(final FSDirectory fsdir, final Path dir)
throws IOException {
final String dirStr = dir.toString();
return INodeDirectory.valueOf(fsdir.getINode(dirStr), dirStr);
}
private void checkQuotaUsageComputation(final Path dirPath, private void checkQuotaUsageComputation(final Path dirPath,
final long expectedNs, final long expectedDs) throws IOException { final long expectedNs, final long expectedDs) throws IOException {
INode node = fsdir.getINode(dirPath.toString()); INodeDirectory dirNode = getDir(fsdir, dirPath);
assertTrue(node.isDirectory() && node.isQuotaSet()); assertTrue(dirNode.isQuotaSet());
INodeDirectoryWithQuota dirNode = (INodeDirectoryWithQuota) node; Quota.Counts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs, assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
dirNode.getNamespace()); q.get(Quota.NAMESPACE));
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs, assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
dirNode.getDiskspace()); q.get(Quota.DISKSPACE));
Quota.Counts counts = Quota.Counts.newInstance(); Quota.Counts counts = Quota.Counts.newInstance();
dirNode.computeQuotaUsage(counts, false); dirNode.computeQuotaUsage(counts, false);
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs, assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,

View File

@ -305,7 +305,7 @@ static void modify(INode inode, final List<INode> current,
final int i = Diff.search(current, inode.getKey()); final int i = Diff.search(current, inode.getKey());
Assert.assertTrue(i >= 0); Assert.assertTrue(i >= 0);
final INodeDirectory oldinode = (INodeDirectory)current.get(i); final INodeDirectory oldinode = (INodeDirectory)current.get(i);
final INodeDirectory newinode = new INodeDirectory(oldinode, false); final INodeDirectory newinode = new INodeDirectory(oldinode, false, true);
newinode.setModificationTime(oldinode.getModificationTime() + 1); newinode.setModificationTime(oldinode.getModificationTime() + 1);
current.set(i, newinode); current.set(i, newinode);