diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 73399348f79..de710e6ebef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -246,6 +246,9 @@ Trunk (Unreleased) HDFS-5629. Support HTTPS in JournalNode and SecondaryNameNode. (Haohui Mai via jing9) + HDFS-5647. Merge INodeDirectory.Feature and INodeFile.Feature. (Haohui Mai + via jing9) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java index c03a7971d11..511b5fdf6b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException; /** * Quota feature for {@link INodeDirectory}. */ -public final class DirectoryWithQuotaFeature extends INodeDirectory.Feature { +public final class DirectoryWithQuotaFeature implements INode.Feature { public static final long DEFAULT_NAMESPACE_QUOTA = Long.MAX_VALUE; public static final long DEFAULT_DISKSPACE_QUOTA = HdfsConstants.QUOTA_RESET; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java index 0a7371dc485..782a859e047 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; * Feature for under-construction file. */ @InterfaceAudience.Private -public class FileUnderConstructionFeature extends INodeFile.Feature { +public class FileUnderConstructionFeature implements INode.Feature { private String clientName; // lease holder private final String clientMachine; // if client is a cluster node too. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 1ba48a012e0..c90369e4a5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -756,47 +756,6 @@ public abstract class INode implements INodeAttributes, Diff.Element { /** INode feature such as {@link FileUnderConstructionFeature} * and {@link DirectoryWithQuotaFeature}. */ - interface Feature> { - /** @return the next feature. */ - public F getNextFeature(); - - /** Set the next feature. */ - public void setNextFeature(F next); - - /** Utility methods such as addFeature and removeFeature. */ - static class Util { - /** - * Add a feature to the linked list. - * @return the new head. - */ - static > F addFeature(F feature, F head) { - feature.setNextFeature(head); - return feature; - } - - /** - * Remove a feature from the linked list. - * @return the new head. - */ - static > F removeFeature(F feature, F head) { - if (feature == head) { - final F newHead = head.getNextFeature(); - head.setNextFeature(null); - return newHead; - } else if (head != null) { - F prev = head; - F curr = head.getNextFeature(); - for (; curr != null && curr != feature; - prev = curr, curr = curr.getNextFeature()) - ; - if (curr != null) { - prev.setNextFeature(curr.getNextFeature()); - curr.setNextFeature(null); - return head; - } - } - throw new IllegalStateException("Feature " + feature + " not found."); - } - } + public interface Feature { } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 05e98673658..71fdc28d810 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -45,20 +45,6 @@ import com.google.common.base.Preconditions; */ public class INodeDirectory extends INodeWithAdditionalFields implements INodeDirectoryAttributes { - /** Directory related features such as quota and snapshots. */ - public static abstract class Feature implements INode.Feature { - private Feature nextFeature; - - @Override - public Feature getNextFeature() { - return nextFeature; - } - - @Override - public void setNextFeature(Feature next) { - this.nextFeature = next; - } - } /** Cast INode to INodeDirectory. */ public static INodeDirectory valueOf(INode inode, Object path @@ -78,9 +64,6 @@ public class INodeDirectory extends INodeWithAdditionalFields private List children = null; - /** A linked list of {@link Feature}s. */ - private Feature headFeature = null; - /** constructor */ public INodeDirectory(long id, byte[] name, PermissionStatus permissions, long mtime) { @@ -102,7 +85,7 @@ public class INodeDirectory extends INodeWithAdditionalFields } } if (copyFeatures) { - this.headFeature = other.headFeature; + this.features = other.features; } } @@ -160,7 +143,7 @@ public class INodeDirectory extends INodeWithAdditionalFields * otherwise, return null. */ public final DirectoryWithQuotaFeature getDirectoryWithQuotaFeature() { - for(Feature f = headFeature; f != null; f = f.nextFeature) { + for (Feature f : features) { if (f instanceof DirectoryWithQuotaFeature) { return (DirectoryWithQuotaFeature)f; } @@ -182,14 +165,6 @@ public class INodeDirectory extends INodeWithAdditionalFields return quota; } - private void addFeature(Feature f) { - headFeature = INode.Feature.Util.addFeature(f, headFeature); - } - - private void removeFeature(Feature f) { - headFeature = INode.Feature.Util.removeFeature(f, headFeature); - } - private int searchChildren(byte[] name) { return children == null? -1: Collections.binarySearch(children, name); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index cced4e7bd30..1d420e84761 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -41,23 +41,6 @@ import com.google.common.base.Preconditions; @InterfaceAudience.Private public class INodeFile extends INodeWithAdditionalFields implements INodeFileAttributes, BlockCollection { - /** - * A feature contains specific information for a type of INodeFile. E.g., - * we can have separate features for Under-Construction and Snapshot. - */ - public static abstract class Feature implements INode.Feature { - private Feature nextFeature; - - @Override - public Feature getNextFeature() { - return nextFeature; - } - - @Override - public void setNextFeature(Feature next) { - this.nextFeature = next; - } - } /** The same as valueOf(inode, path, false). */ public static INodeFile valueOf(INode inode, String path @@ -120,8 +103,6 @@ public class INodeFile extends INodeWithAdditionalFields private BlockInfo[] blocks; - private Feature headFeature; - INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime, long atime, BlockInfo[] blklist, short replication, long preferredBlockSize) { @@ -135,7 +116,7 @@ public class INodeFile extends INodeWithAdditionalFields super(that); this.header = that.header; this.blocks = that.blocks; - this.headFeature = that.headFeature; + this.features = that.features; } public INodeFile(INodeFile that, FileDiffList diffs) { @@ -144,14 +125,6 @@ public class INodeFile extends INodeWithAdditionalFields this.addSnapshotFeature(diffs); } - private void addFeature(Feature f) { - headFeature = INode.Feature.Util.addFeature(f, headFeature); - } - - private void removeFeature(Feature f) { - headFeature = INode.Feature.Util.removeFeature(f, headFeature); - } - /** @return true unconditionally. */ @Override public final boolean isFile() { @@ -171,7 +144,7 @@ public class INodeFile extends INodeWithAdditionalFields * otherwise, return null. */ public final FileUnderConstructionFeature getFileUnderConstructionFeature() { - for (Feature f = this.headFeature; f != null; f = f.nextFeature) { + for (Feature f : features) { if (f instanceof FileUnderConstructionFeature) { return (FileUnderConstructionFeature) f; } @@ -283,7 +256,7 @@ public class INodeFile extends INodeWithAdditionalFields * otherwise, return null. */ public final FileWithSnapshotFeature getFileWithSnapshotFeature() { - for (Feature f = headFeature; f != null; f = f.nextFeature) { + for (Feature f: features) { if (f instanceof FileWithSnapshotFeature) { return (FileWithSnapshotFeature) f; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java index f8b63ec2265..bc74ad2a10b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java @@ -95,6 +95,9 @@ public abstract class INodeWithAdditionalFields extends INode /** For implementing {@link LinkedElement}. */ private LinkedElement next = null; + /** An array {@link Feature}s. */ + private static final Feature[] EMPTY_FEATURE = new Feature[0]; + protected Feature[] features = EMPTY_FEATURE; private INodeWithAdditionalFields(INode parent, long id, byte[] name, long permission, long modificationTime, long accessTime) { @@ -262,4 +265,45 @@ public abstract class INodeWithAdditionalFields extends INode public final void setAccessTime(long accessTime) { this.accessTime = accessTime; } + + protected void addFeature(Feature f) { + int size = features.length; + Feature[] arr = new Feature[size + 1]; + if (size != 0) { + System.arraycopy(features, 0, arr, 0, size); + } + arr[size] = f; + features = arr; + } + + protected void removeFeature(Feature f) { + int size = features.length; + Preconditions.checkState(size > 0, "Feature " + + f.getClass().getSimpleName() + " not found."); + + if (size == 1) { + Preconditions.checkState(features[0] == f, "Feature " + + f.getClass().getSimpleName() + " not found."); + features = EMPTY_FEATURE; + return; + } + + Feature[] arr = new Feature[size - 1]; + int j = 0; + boolean overflow = false; + for (Feature f1 : features) { + if (f1 != f) { + if (j == size - 1) { + overflow = true; + break; + } else { + arr[j++] = f1; + } + } + } + + Preconditions.checkState(!overflow && j == size - 1, "Feature " + + f.getClass().getSimpleName() + " not found."); + features = arr; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java index 737603a45bd..b74d2694789 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.server.namenode.Quota; * Feature for file with snapshot-related information. */ @InterfaceAudience.Private -public class FileWithSnapshotFeature extends INodeFile.Feature { +public class FileWithSnapshotFeature implements INode.Feature { private final FileDiffList diffs; private boolean isCurrentFileDeleted = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index 2f921907927..642599d09fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Time; import org.junit.Test; import org.mockito.Mockito; @@ -921,6 +922,7 @@ public class TestINodeFile { public void testDotdotInodePath() throws Exception { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; + DFSClient client = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); @@ -933,7 +935,7 @@ public class TestINodeFile { long parentId = fsdir.getINode("/").getId(); String testPath = "/.reserved/.inodes/" + dirId + "/.."; - DFSClient client = new DFSClient(NameNode.getAddress(conf), conf); + client = new DFSClient(NameNode.getAddress(conf), conf); HdfsFileStatus status = client.getFileInfo(testPath); assertTrue(parentId == status.getFileId()); @@ -943,6 +945,7 @@ public class TestINodeFile { assertTrue(parentId == status.getFileId()); } finally { + IOUtils.cleanup(LOG, client); if (cluster != null) { cluster.shutdown(); }