From 25c508e4251b2e99ba9560f358b5d09b62ec8912 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sat, 15 Feb 2014 00:08:04 +0000 Subject: [PATCH] HDFS-5698. Merge r1566359 from trunk for merging the 'Use protobuf to serialize / deserialize FSImage' feature git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1568548 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 52 ++ .../dev-support/findbugsExcludeFile.xml | 3 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + .../hadoop-hdfs/src/main/bin/hdfs | 2 +- .../hadoop/hdfs/protocol/LayoutVersion.java | 3 +- .../DelegationTokenSecretManager.java | 87 ++- .../hdfs/server/namenode/CacheManager.java | 147 ++++- .../hadoop/hdfs/server/namenode/FSImage.java | 5 +- .../server/namenode/FSImageCompression.java | 6 +- .../hdfs/server/namenode/FSImageFormat.java | 75 ++- .../server/namenode/FSImageFormatPBINode.java | 466 +++++++++++++++ .../namenode/FSImageFormatProtobuf.java | 551 ++++++++++++++++++ .../hdfs/server/namenode/FSImageUtil.java | 93 +++ .../hdfs/server/namenode/FSNamesystem.java | 22 + .../hdfs/server/namenode/INodeDirectory.java | 2 +- .../hdfs/server/namenode/INodeFile.java | 2 +- .../hadoop/hdfs/server/namenode/INodeMap.java | 5 + .../server/namenode/SaveNamespaceContext.java | 6 +- .../DirectoryWithSnapshotFeature.java | 6 +- .../snapshot/FSImageFormatPBSnapshot.java | 437 ++++++++++++++ .../snapshot/SnapshotFSImageFormat.java | 26 +- .../namenode/snapshot/SnapshotManager.java | 17 + .../FileDistributionCalculator.java | 160 +++++ .../ImageLoaderCurrent.java | 2 +- .../tools/offlineImageViewer/LsrPBImage.java | 233 ++++++++ .../OfflineImageViewerPB.java | 178 ++++++ .../offlineImageViewer/PBImageXmlWriter.java | 415 +++++++++++++ .../hadoop-hdfs/src/main/proto/fsimage.proto | 280 +++++++++ .../hdfs/server/namenode/TestFSImage.java | 138 +++++ .../namenode/TestFSImageWithSnapshot.java | 4 +- .../namenode/ha/TestStandbyCheckpoints.java | 2 +- .../snapshot/TestRenameWithSnapshots.java | 1 - .../namenode/snapshot/TestSnapshot.java | 39 +- .../TestOfflineImageViewer.java | 427 ++++---------- .../src/test/resources/editsStored.xml | 2 +- 35 files changed, 3491 insertions(+), 404 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a5032634dde..9a36fccbdb5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -69,6 +69,9 @@ Release 2.4.0 - UNRELEASED HDFS-5775. Consolidate the code for serialization in CacheManager (Haohui Mai via brandonli) + HDFS-5768. Consolidate the serialization code in DelegationTokenSecretManager + (Haohui Mai via brandonli) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery @@ -152,6 +155,55 @@ Release 2.4.0 - UNRELEASED HDFS-5949. New Namenode UI when trying to download a file, the browser doesn't know the file name. (Haohui Mai via brandonli) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS + + HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) + + HDFS-5738. Serialize INode information in protobuf. (Haohui Mai via jing9) + + HDFS-5772. Serialize under-construction file information in FSImage. (jing9) + + HDFS-5783. Compute the digest before loading FSImage. (Haohui Mai via jing9) + + HDFS-5785. Serialize symlink in protobuf. (Haohui Mai via jing9) + + HDFS-5793. Optimize the serialization of PermissionStatus. (Haohui Mai via + jing9) + + HDFS-5743. Use protobuf to serialize snapshot information. (jing9) + + HDFS-5774. Serialize CachePool directives in protobuf. (Haohui Mai via jing9) + + HDFS-5744. Serialize information for token managers in protobuf. (Haohui Mai + via jing9) + + HDFS-5824. Add a Type field in Snapshot DiffEntry's protobuf definition. + (jing9) + + HDFS-5808. Implement cancellation when saving FSImage. (Haohui Mai via jing9) + + HDFS-5826. Update the stored edit logs to be consistent with the changes in + HDFS-5698 branch. (Haohui Mai via jing9) + + HDFS-5797. Implement offline image viewer. (Haohui Mai via jing9) + + HDFS-5771. Track progress when loading fsimage. (Haohui Mai via cnauroth) + + HDFS-5871. Use PBHelper to serialize CacheDirectiveInfoExpirationProto. + (Haohui Mai via jing9) + + HDFS-5884. LoadDelegator should use IOUtils.readFully() to read the magic + header. (Haohui Mai via jing9) + + HDFS-5885. Add annotation for repeated fields in the protobuf definition. + (Haohui Mai via jing9) + + HDFS-5906. Fixing findbugs and javadoc warnings in the HDFS-5698 branch. + (Haohui Mai via jing9) + + HDFS-5911. The id of a CacheDirective instance does not get serialized in + the protobuf-fsimage. (Haohui Mai via jing9) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml index f97110705ac..e3c63b2be8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml @@ -8,6 +8,9 @@ + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 159b5ae501b..b9107a35ad6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -453,6 +453,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> ClientDatanodeProtocol.proto DatanodeProtocol.proto + fsimage.proto ${project.build.directory}/generated-sources/java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 91c8c9af50d..16f2e2ab477 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -139,7 +139,7 @@ elif [ "$COMMAND" = "balancer" ] ; then elif [ "$COMMAND" = "jmxget" ] ; then CLASS=org.apache.hadoop.hdfs.tools.JMXGet elif [ "$COMMAND" = "oiv" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer + CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB elif [ "$COMMAND" = "oev" ] ; then CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer elif [ "$COMMAND" = "fetchdt" ] ; then diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java index 923ed70ac8f..9842b53fbd3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java @@ -112,7 +112,8 @@ public class LayoutVersion { ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid." + " Use distinct StorageUuid per storage directory."), ADD_LAYOUT_FLAGS(-50, "Add support for layout flags."), - CACHING(-51, "Support for cache pools and path-based caching"); + CACHING(-51, "Support for cache pools and path-based caching"), + PROTOBUF_FORMAT(-52, "Use protobuf to serialize FSImage"); final int lv; final int ancestorLV; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java index e291204cc23..b9fce60446b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java @@ -23,12 +23,16 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; @@ -46,6 +50,10 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; + /** * A HDFS specific delegation token secret manager. * The secret manager is responsible for generating and accepting the password @@ -167,7 +175,45 @@ public class DelegationTokenSecretManager } serializerCompat.load(in); } - + + public static class SecretManagerState { + public final SecretManagerSection section; + public final List keys; + public final List tokens; + + public SecretManagerState( + SecretManagerSection s, + List keys, + List tokens) { + this.section = s; + this.keys = keys; + this.tokens = tokens; + } + } + + public synchronized void loadSecretManagerState(SecretManagerState state) + throws IOException { + Preconditions.checkState(!running, + "Can't load state from image in a running SecretManager."); + + currentId = state.section.getCurrentId(); + delegationTokenSequenceNumber = state.section.getTokenSequenceNumber(); + for (SecretManagerSection.DelegationKey k : state.keys) { + addKey(new DelegationKey(k.getId(), k.getExpiryDate(), k.hasKey() ? k + .getKey().toByteArray() : null)); + } + + for (SecretManagerSection.PersistToken t : state.tokens) { + DelegationTokenIdentifier id = new DelegationTokenIdentifier(new Text( + t.getOwner()), new Text(t.getRenewer()), new Text(t.getRealUser())); + id.setIssueDate(t.getIssueDate()); + id.setMaxDate(t.getMaxDate()); + id.setSequenceNumber(t.getSequenceNumber()); + id.setMasterKeyId(t.getMasterKeyId()); + addPersistedDelegationToken(id, t.getExpiryDate()); + } + } + /** * Store the current state of the SecretManager for persistence * @@ -179,7 +225,43 @@ public class DelegationTokenSecretManager String sdPath) throws IOException { serializerCompat.save(out, sdPath); } - + + public synchronized SecretManagerState saveSecretManagerState() { + SecretManagerSection s = SecretManagerSection.newBuilder() + .setCurrentId(currentId) + .setTokenSequenceNumber(delegationTokenSequenceNumber) + .setNumKeys(allKeys.size()).setNumTokens(currentTokens.size()).build(); + ArrayList keys = Lists + .newArrayListWithCapacity(allKeys.size()); + ArrayList tokens = Lists + .newArrayListWithCapacity(currentTokens.size()); + + for (DelegationKey v : allKeys.values()) { + SecretManagerSection.DelegationKey.Builder b = SecretManagerSection.DelegationKey + .newBuilder().setId(v.getKeyId()).setExpiryDate(v.getExpiryDate()); + if (v.getEncodedKey() != null) { + b.setKey(ByteString.copyFrom(v.getEncodedKey())); + } + keys.add(b.build()); + } + + for (Entry e : currentTokens + .entrySet()) { + DelegationTokenIdentifier id = e.getKey(); + SecretManagerSection.PersistToken.Builder b = SecretManagerSection.PersistToken + .newBuilder().setOwner(id.getOwner().toString()) + .setRenewer(id.getRenewer().toString()) + .setRealUser(id.getRealUser().toString()) + .setIssueDate(id.getIssueDate()).setMaxDate(id.getMaxDate()) + .setSequenceNumber(id.getSequenceNumber()) + .setMasterKeyId(id.getMasterKeyId()) + .setExpiryDate(e.getValue().getRenewDate()); + tokens.add(b.build()); + } + + return new SecretManagerState(s, keys, tokens); + } + /** * This method is intended to be used only while reading edit logs. * @@ -431,4 +513,5 @@ public class DelegationTokenSecretManager prog.endStep(Phase.LOADING_FSIMAGE, step); } } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index ba3936ca997..de536b30d79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -50,8 +50,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.InvalidRequestException; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.CacheDirective; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; @@ -62,11 +64,15 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; +import org.apache.hadoop.hdfs.protocolPB.PBHelper; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; @@ -81,6 +87,7 @@ import org.apache.hadoop.util.LightWeightGSet; import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; /** * The Cache Manager handles caching on DataNodes. @@ -167,6 +174,19 @@ public final class CacheManager { */ private CacheReplicationMonitor monitor; + public static final class PersistState { + public final CacheManagerSection section; + public final List pools; + public final List directives; + + public PersistState(CacheManagerSection section, + List pools, List directives) { + this.section = section; + this.pools = pools; + this.directives = directives; + } + } + CacheManager(FSNamesystem namesystem, Configuration conf, BlockManager blockManager) { this.namesystem = namesystem; @@ -944,6 +964,64 @@ public final class CacheManager { serializerCompat.save(out, sdPath); } + public PersistState saveState() throws IOException { + ArrayList pools = Lists + .newArrayListWithCapacity(cachePools.size()); + ArrayList directives = Lists + .newArrayListWithCapacity(directivesById.size()); + + for (CachePool pool : cachePools.values()) { + CachePoolInfo p = pool.getInfo(true); + CachePoolInfoProto.Builder b = CachePoolInfoProto.newBuilder() + .setPoolName(p.getPoolName()); + + if (p.getOwnerName() != null) + b.setOwnerName(p.getOwnerName()); + + if (p.getGroupName() != null) + b.setGroupName(p.getGroupName()); + + if (p.getMode() != null) + b.setMode(p.getMode().toShort()); + + if (p.getLimit() != null) + b.setLimit(p.getLimit()); + + pools.add(b.build()); + } + + for (CacheDirective directive : directivesById.values()) { + CacheDirectiveInfo info = directive.toInfo(); + CacheDirectiveInfoProto.Builder b = CacheDirectiveInfoProto.newBuilder() + .setId(info.getId()); + + if (info.getPath() != null) { + b.setPath(info.getPath().toUri().getPath()); + } + + if (info.getReplication() != null) { + b.setReplication(info.getReplication()); + } + + if (info.getPool() != null) { + b.setPool(info.getPool()); + } + + Expiration expiry = info.getExpiration(); + if (expiry != null) { + assert (!expiry.isRelative()); + b.setExpiration(PBHelper.convert(expiry)); + } + + directives.add(b.build()); + } + CacheManagerSection s = CacheManagerSection.newBuilder() + .setNextDirectiveId(nextDirectiveId).setNumPools(pools.size()) + .setNumDirectives(directives.size()).build(); + + return new PersistState(s, pools, directives); + } + /** * Reloads CacheManager state from the passed DataInput. Used during namenode * startup to restore CacheManager state from an FSImage. @@ -954,6 +1032,56 @@ public final class CacheManager { serializerCompat.load(in); } + public void loadState(PersistState s) throws IOException { + nextDirectiveId = s.section.getNextDirectiveId(); + for (CachePoolInfoProto p : s.pools) { + CachePoolInfo info = new CachePoolInfo(p.getPoolName()); + if (p.hasOwnerName()) + info.setOwnerName(p.getOwnerName()); + + if (p.hasGroupName()) + info.setGroupName(p.getGroupName()); + + if (p.hasMode()) + info.setMode(new FsPermission((short) p.getMode())); + + if (p.hasLimit()) + info.setLimit(p.getLimit()); + + addCachePool(info); + } + + for (CacheDirectiveInfoProto p : s.directives) { + // Get pool reference by looking it up in the map + final String poolName = p.getPool(); + CacheDirective directive = new CacheDirective(p.getId(), new Path( + p.getPath()).toUri().getPath(), (short) p.getReplication(), p + .getExpiration().getMillis()); + addCacheDirective(poolName, directive); + } + } + + private void addCacheDirective(final String poolName, + final CacheDirective directive) throws IOException { + CachePool pool = cachePools.get(poolName); + if (pool == null) { + throw new IOException("Directive refers to pool " + poolName + + ", which does not exist."); + } + boolean addedDirective = pool.getDirectiveList().add(directive); + assert addedDirective; + if (directivesById.put(directive.getId(), directive) != null) { + throw new IOException("A directive with ID " + directive.getId() + + " already exists"); + } + List directives = directivesByPath.get(directive.getPath()); + if (directives == null) { + directives = new LinkedList(); + directivesByPath.put(directive.getPath(), directives); + } + directives.add(directive); + } + private final class SerializerCompat { private void save(DataOutputStream out, String sdPath) throws IOException { out.writeLong(nextDirectiveId); @@ -1036,27 +1164,10 @@ public final class CacheManager { CacheDirectiveInfo info = FSImageSerialization.readCacheDirectiveInfo(in); // Get pool reference by looking it up in the map final String poolName = info.getPool(); - CachePool pool = cachePools.get(poolName); - if (pool == null) { - throw new IOException("Directive refers to pool " + poolName + - ", which does not exist."); - } CacheDirective directive = new CacheDirective(info.getId(), info.getPath().toUri().getPath(), info.getReplication(), info.getExpiration().getAbsoluteMillis()); - boolean addedDirective = pool.getDirectiveList().add(directive); - assert addedDirective; - if (directivesById.put(directive.getId(), directive) != null) { - throw new IOException("A directive with ID " + directive.getId() + - " already exists"); - } - List directives = - directivesByPath.get(directive.getPath()); - if (directives == null) { - directives = new LinkedList(); - directivesByPath.put(directive.getPath(), directives); - } - directives.add(directive); + addCacheDirective(poolName, directive); counter.increment(); } prog.endStep(Phase.LOADING_FSIMAGE, step); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 0a3b49060b5..d1cbbb58d73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -827,8 +827,7 @@ public class FSImage implements Closeable { */ private void loadFSImage(File curFile, MD5Hash expectedMd5, FSNamesystem target, MetaRecoveryContext recovery) throws IOException { - FSImageFormat.Loader loader = new FSImageFormat.Loader( - conf, target); + FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, target); loader.load(curFile); target.setBlockPoolId(this.getBlockPoolID()); @@ -857,7 +856,7 @@ public class FSImage implements Closeable { File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid); File dstFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE, txid); - FSImageFormat.Saver saver = new FSImageFormat.Saver(context); + FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); saver.save(newFile, compression); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java index e0a46f15445..872ee74c802 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageCompression.java @@ -57,6 +57,10 @@ class FSImageCompression { imageCodec = codec; } + public CompressionCodec getImageCodec() { + return imageCodec; + } + /** * Create a "noop" compression - i.e. uncompressed */ @@ -89,7 +93,7 @@ class FSImageCompression { * Create a compression instance using the codec specified by * codecClassName */ - private static FSImageCompression createCompression(Configuration conf, + static FSImageCompression createCompression(Configuration conf, String codecClassName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index dca9888322d..7b45b0870ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -69,12 +69,13 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Co import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.util.ReadOnlyList; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; import org.apache.hadoop.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.annotations.VisibleForTesting; /** * Contains inner classes for reading or writing the on-disk format for @@ -181,16 +182,74 @@ import com.google.common.base.Preconditions; @InterfaceStability.Evolving public class FSImageFormat { private static final Log LOG = FSImage.LOG; - + // Static-only class private FSImageFormat() {} - + + interface AbstractLoader { + MD5Hash getLoadedImageMd5(); + long getLoadedImageTxId(); + } + + static class LoaderDelegator implements AbstractLoader { + private AbstractLoader impl; + private final Configuration conf; + private final FSNamesystem fsn; + + LoaderDelegator(Configuration conf, FSNamesystem fsn) { + this.conf = conf; + this.fsn = fsn; + } + + @Override + public MD5Hash getLoadedImageMd5() { + return impl.getLoadedImageMd5(); + } + + @Override + public long getLoadedImageTxId() { + return impl.getLoadedImageTxId(); + } + + public void load(File file) throws IOException { + Preconditions.checkState(impl == null, "Image already loaded!"); + + FileInputStream is = null; + try { + is = new FileInputStream(file); + byte[] magic = new byte[FSImageUtil.MAGIC_HEADER.length]; + IOUtils.readFully(is, magic, 0, magic.length); + if (Arrays.equals(magic, FSImageUtil.MAGIC_HEADER)) { + FSImageFormatProtobuf.Loader loader = new FSImageFormatProtobuf.Loader( + conf, fsn); + impl = loader; + loader.load(file); + } else { + Loader loader = new Loader(conf, fsn); + impl = loader; + loader.load(file); + } + + } finally { + IOUtils.cleanup(LOG, is); + } + } + } + + /** + * Construct a loader class to load the image. It chooses the loader based on + * the layout version. + */ + public static LoaderDelegator newLoader(Configuration conf, FSNamesystem fsn) { + return new LoaderDelegator(conf, fsn); + } + /** * A one-shot class responsible for loading an image. The load() function * should be called once, after which the getter methods may be used to retrieve * information about the image that was loaded, if loading was successful. */ - public static class Loader { + public static class Loader implements AbstractLoader { private final Configuration conf; /** which namesystem this loader is working for */ private final FSNamesystem namesystem; @@ -215,12 +274,14 @@ public class FSImageFormat { * Return the MD5 checksum of the image that has been loaded. * @throws IllegalStateException if load() has not yet been called. */ - MD5Hash getLoadedImageMd5() { + @Override + public MD5Hash getLoadedImageMd5() { checkLoaded(); return imgDigest; } - long getLoadedImageTxId() { + @Override + public long getLoadedImageTxId() { checkLoaded(); return imgTxId; } @@ -243,7 +304,7 @@ public class FSImageFormat { } } - void load(File curFile) throws IOException { + public void load(File curFile) throws IOException { checkNotLoaded(); assert curFile != null : "curFile is null"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java new file mode 100644 index 00000000000..5ade5cec6a3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -0,0 +1,466 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; +import org.apache.hadoop.hdfs.protocolPB.PBHelper; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.StringMap; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; +import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference; +import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; +import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.util.ReadOnlyList; + +import com.google.common.base.Preconditions; +import com.google.protobuf.ByteString; + +@InterfaceAudience.Private +public final class FSImageFormatPBINode { + private final static long USER_GROUP_STRID_MASK = (1 << 24) - 1; + private final static int USER_STRID_OFFSET = 40; + private final static int GROUP_STRID_OFFSET = 16; + private static final Log LOG = LogFactory.getLog(FSImageFormatProtobuf.class); + + public final static class Loader { + public static PermissionStatus loadPermission(long id, + final String[] stringTable) { + short perm = (short) (id & ((1 << GROUP_STRID_OFFSET) - 1)); + int gsid = (int) ((id >> GROUP_STRID_OFFSET) & USER_GROUP_STRID_MASK); + int usid = (int) ((id >> USER_STRID_OFFSET) & USER_GROUP_STRID_MASK); + return new PermissionStatus(stringTable[usid], stringTable[gsid], + new FsPermission(perm)); + } + + public static INodeReference loadINodeReference( + INodeSection.INodeReference r, FSDirectory dir) throws IOException { + long referredId = r.getReferredId(); + INode referred = dir.getInode(referredId); + WithCount withCount = (WithCount) referred.getParentReference(); + if (withCount == null) { + withCount = new INodeReference.WithCount(null, referred); + } + final INodeReference ref; + if (r.hasDstSnapshotId()) { // DstReference + ref = new INodeReference.DstReference(null, withCount, + r.getDstSnapshotId()); + } else { + ref = new INodeReference.WithName(null, withCount, r.getName() + .toByteArray(), r.getLastSnapshotId()); + } + return ref; + } + + public static INodeDirectory loadINodeDirectory(INodeSection.INode n, + final String[] stringTable) { + assert n.getType() == INodeSection.INode.Type.DIRECTORY; + INodeSection.INodeDirectory d = n.getDirectory(); + + final PermissionStatus permissions = loadPermission(d.getPermission(), + stringTable); + final INodeDirectory dir = new INodeDirectory(n.getId(), n.getName() + .toByteArray(), permissions, d.getModificationTime()); + + final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota(); + if (nsQuota >= 0 || dsQuota >= 0) { + dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota); + } + return dir; + } + + public static void updateBlocksMap(INodeFile file, BlockManager bm) { + // Add file->block mapping + final BlockInfo[] blocks = file.getBlocks(); + if (blocks != null) { + for (int i = 0; i < blocks.length; i++) { + file.setBlock(i, bm.addBlockCollection(blocks[i], file)); + } + } + } + + private final FSDirectory dir; + private final FSNamesystem fsn; + private final FSImageFormatProtobuf.Loader parent; + + Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) { + this.fsn = fsn; + this.dir = fsn.dir; + this.parent = parent; + } + + void loadINodeDirectorySection(InputStream in) throws IOException { + while (true) { + INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry + .parseDelimitedFrom(in); + // note that in is a LimitedInputStream + if (e == null) { + break; + } + INodeDirectory p = dir.getInode(e.getParent()).asDirectory(); + for (long id : e.getChildrenList()) { + INode child = dir.getInode(id); + addToParent(p, child); + } + for (int i = 0; i < e.getNumOfRef(); i++) { + INodeReference ref = loadINodeReference(in); + addToParent(p, ref); + } + } + } + + private INodeReference loadINodeReference(InputStream in) + throws IOException { + INodeSection.INodeReference ref = INodeSection.INodeReference + .parseDelimitedFrom(in); + return loadINodeReference(ref, dir); + } + + void loadINodeSection(InputStream in) throws IOException { + INodeSection s = INodeSection.parseDelimitedFrom(in); + fsn.resetLastInodeId(s.getLastInodeId()); + LOG.info("Loading " + s.getNumInodes() + " INodes."); + for (int i = 0; i < s.getNumInodes(); ++i) { + INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); + if (p.getId() == INodeId.ROOT_INODE_ID) { + loadRootINode(p); + } else { + INode n = loadINode(p); + dir.addToInodeMap(n); + } + } + } + + /** + * Load the under-construction files section, and update the lease map + */ + void loadFilesUnderConstructionSection(InputStream in) throws IOException { + while (true) { + FileUnderConstructionEntry entry = FileUnderConstructionEntry + .parseDelimitedFrom(in); + if (entry == null) { + break; + } + // update the lease manager + INodeFile file = dir.getInode(entry.getInodeId()).asFile(); + FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature(); + Preconditions.checkState(uc != null); // file must be under-construction + fsn.leaseManager.addLease(uc.getClientName(), entry.getFullPath()); + } + } + + private void addToParent(INodeDirectory parent, INode child) { + if (parent == dir.rootDir && FSDirectory.isReservedName(child)) { + throw new HadoopIllegalArgumentException("File name \"" + + child.getLocalName() + "\" is reserved. Please " + + " change the name of the existing file or directory to another " + + "name before upgrading to this release."); + } + // NOTE: This does not update space counts for parents + if (!parent.addChild(child)) { + return; + } + dir.cacheName(child); + + if (child.isFile()) { + updateBlocksMap(child.asFile(), fsn.getBlockManager()); + } + } + + private INode loadINode(INodeSection.INode n) { + switch (n.getType()) { + case FILE: + return loadINodeFile(n); + case DIRECTORY: + return loadINodeDirectory(n, parent.getStringTable()); + case SYMLINK: + return loadINodeSymlink(n); + default: + break; + } + return null; + } + + private INodeFile loadINodeFile(INodeSection.INode n) { + assert n.getType() == INodeSection.INode.Type.FILE; + INodeSection.INodeFile f = n.getFile(); + List bp = f.getBlocksList(); + short replication = (short) f.getReplication(); + + BlockInfo[] blocks = new BlockInfo[bp.size()]; + for (int i = 0, e = bp.size(); i < e; ++i) { + blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication); + } + final PermissionStatus permissions = loadPermission(f.getPermission(), + parent.getStringTable()); + + final INodeFile file = new INodeFile(n.getId(), + n.getName().toByteArray(), permissions, f.getModificationTime(), + f.getAccessTime(), blocks, replication, f.getPreferredBlockSize()); + // under-construction information + if (f.hasFileUC()) { + INodeSection.FileUnderConstructionFeature uc = f.getFileUC(); + file.toUnderConstruction(uc.getClientName(), uc.getClientMachine(), + null); + if (blocks.length > 0) { + BlockInfo lastBlk = file.getLastBlock(); + // replace the last block of file + file.setBlock(file.numBlocks() - 1, new BlockInfoUnderConstruction( + lastBlk, replication)); + } + } + return file; + } + + + private INodeSymlink loadINodeSymlink(INodeSection.INode n) { + assert n.getType() == INodeSection.INode.Type.SYMLINK; + INodeSection.INodeSymlink s = n.getSymlink(); + final PermissionStatus permissions = loadPermission(s.getPermission(), + parent.getStringTable()); + return new INodeSymlink(n.getId(), n.getName().toByteArray(), permissions, + 0, 0, s.getTarget().toStringUtf8()); + } + + private void loadRootINode(INodeSection.INode p) { + INodeDirectory root = loadINodeDirectory(p, parent.getStringTable()); + final Quota.Counts q = root.getQuotaCounts(); + final long nsQuota = q.get(Quota.NAMESPACE); + final long dsQuota = q.get(Quota.DISKSPACE); + if (nsQuota != -1 || dsQuota != -1) { + dir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota); + } + dir.rootDir.cloneModificationTime(root); + dir.rootDir.clonePermissionStatus(root); + } + } + + public final static class Saver { + private static long buildPermissionStatus(INodeAttributes n, + final StringMap stringMap) { + long userId = stringMap.getStringId(n.getUserName()); + long groupId = stringMap.getStringId(n.getGroupName()); + return ((userId & USER_GROUP_STRID_MASK) << USER_STRID_OFFSET) + | ((groupId & USER_GROUP_STRID_MASK) << GROUP_STRID_OFFSET) + | n.getFsPermissionShort(); + } + + public static INodeSection.INodeFile.Builder buildINodeFile( + INodeFileAttributes file, final StringMap stringMap) { + INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder() + .setAccessTime(file.getAccessTime()) + .setModificationTime(file.getModificationTime()) + .setPermission(buildPermissionStatus(file, stringMap)) + .setPreferredBlockSize(file.getPreferredBlockSize()) + .setReplication(file.getFileReplication()); + return b; + } + + public static INodeSection.INodeDirectory.Builder buildINodeDirectory( + INodeDirectoryAttributes dir, final StringMap stringMap) { + Quota.Counts quota = dir.getQuotaCounts(); + INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory + .newBuilder().setModificationTime(dir.getModificationTime()) + .setNsQuota(quota.get(Quota.NAMESPACE)) + .setDsQuota(quota.get(Quota.DISKSPACE)) + .setPermission(buildPermissionStatus(dir, stringMap)); + return b; + } + + public static INodeSection.INodeReference.Builder buildINodeReference( + INodeReference ref) throws IOException { + INodeSection.INodeReference.Builder rb = INodeSection.INodeReference + .newBuilder().setReferredId(ref.getId()); + if (ref instanceof WithName) { + rb.setLastSnapshotId(((WithName) ref).getLastSnapshotId()).setName( + ByteString.copyFrom(ref.getLocalNameBytes())); + } else if (ref instanceof DstReference) { + rb.setDstSnapshotId(((DstReference) ref).getDstSnapshotId()); + } + return rb; + } + + private final FSNamesystem fsn; + private final FileSummary.Builder summary; + private final SaveNamespaceContext context; + private final FSImageFormatProtobuf.Saver parent; + + Saver(FSImageFormatProtobuf.Saver parent, FileSummary.Builder summary) { + this.parent = parent; + this.summary = summary; + this.context = parent.getContext(); + this.fsn = context.getSourceNamesystem(); + } + + void serializeINodeDirectorySection(OutputStream out) throws IOException { + Iterator iter = fsn.getFSDirectory() + .getINodeMap().getMapIterator(); + int i = 0; + while (iter.hasNext()) { + INodeWithAdditionalFields n = iter.next(); + if (!n.isDirectory()) { + continue; + } + + ReadOnlyList children = n.asDirectory().getChildrenList( + Snapshot.CURRENT_STATE_ID); + if (children.size() > 0) { + INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection. + DirEntry.newBuilder().setParent(n.getId()); + List refs = new ArrayList(); + for (INode inode : children) { + if (!inode.isReference()) { + b.addChildren(inode.getId()); + } else { + refs.add(inode.asReference()); + } + } + b.setNumOfRef(refs.size()); + INodeDirectorySection.DirEntry e = b.build(); + e.writeDelimitedTo(out); + for (INodeReference ref : refs) { + INodeSection.INodeReference.Builder rb = buildINodeReference(ref); + rb.build().writeDelimitedTo(out); + } + } + + ++i; + if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { + context.checkCancelled(); + } + } + parent.commitSection(summary, + FSImageFormatProtobuf.SectionName.INODE_DIR); + } + + void serializeINodeSection(OutputStream out) throws IOException { + INodeMap inodesMap = fsn.dir.getINodeMap(); + + INodeSection.Builder b = INodeSection.newBuilder() + .setLastInodeId(fsn.getLastInodeId()).setNumInodes(inodesMap.size()); + INodeSection s = b.build(); + s.writeDelimitedTo(out); + + int i = 0; + Iterator iter = inodesMap.getMapIterator(); + while (iter.hasNext()) { + INodeWithAdditionalFields n = iter.next(); + save(out, n); + ++i; + if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { + context.checkCancelled(); + } + } + parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE); + } + + void serializeFilesUCSection(OutputStream out) throws IOException { + Map ucMap = fsn.getFilesUnderConstruction(); + for (Map.Entry entry : ucMap.entrySet()) { + String path = entry.getKey(); + INodeFile file = entry.getValue(); + FileUnderConstructionEntry.Builder b = FileUnderConstructionEntry + .newBuilder().setInodeId(file.getId()).setFullPath(path); + FileUnderConstructionEntry e = b.build(); + e.writeDelimitedTo(out); + } + parent.commitSection(summary, + FSImageFormatProtobuf.SectionName.FILES_UNDERCONSTRUCTION); + } + + private void save(OutputStream out, INode n) throws IOException { + if (n.isDirectory()) { + save(out, n.asDirectory()); + } else if (n.isFile()) { + save(out, n.asFile()); + } else if (n.isSymlink()) { + save(out, n.asSymlink()); + } + } + + private void save(OutputStream out, INodeDirectory n) throws IOException { + INodeSection.INodeDirectory.Builder b = buildINodeDirectory(n, + parent.getStringMap()); + INodeSection.INode r = buildINodeCommon(n) + .setType(INodeSection.INode.Type.DIRECTORY).setDirectory(b).build(); + r.writeDelimitedTo(out); + } + + private void save(OutputStream out, INodeFile n) throws IOException { + INodeSection.INodeFile.Builder b = buildINodeFile(n, + parent.getStringMap()); + + for (Block block : n.getBlocks()) { + b.addBlocks(PBHelper.convert(block)); + } + + FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature(); + if (uc != null) { + INodeSection.FileUnderConstructionFeature f = + INodeSection.FileUnderConstructionFeature + .newBuilder().setClientName(uc.getClientName()) + .setClientMachine(uc.getClientMachine()).build(); + b.setFileUC(f); + } + + INodeSection.INode r = buildINodeCommon(n) + .setType(INodeSection.INode.Type.FILE).setFile(b).build(); + r.writeDelimitedTo(out); + } + + private void save(OutputStream out, INodeSymlink n) throws IOException { + INodeSection.INodeSymlink.Builder b = INodeSection.INodeSymlink + .newBuilder() + .setPermission(buildPermissionStatus(n, parent.getStringMap())) + .setTarget(ByteString.copyFrom(n.getSymlink())); + INodeSection.INode r = buildINodeCommon(n) + .setType(INodeSection.INode.Type.SYMLINK).setSymlink(b).build(); + r.writeDelimitedTo(out); + } + + private final INodeSection.INode.Builder buildINodeCommon(INode n) { + return INodeSection.INode.newBuilder() + .setId(n.getId()) + .setName(ByteString.copyFrom(n.getLocalNameBytes())); + } + } + + private FSImageFormatPBINode() { + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java new file mode 100644 index 00000000000..2edc57b18d7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -0,0 +1,551 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; +import org.apache.hadoop.hdfs.server.namenode.snapshot.FSImageFormatPBSnapshot; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; +import org.apache.hadoop.hdfs.util.MD5FileUtils; +import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressorStream; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.io.LimitInputStream; +import com.google.protobuf.CodedOutputStream; + +/** + * Utility class to read / write fsimage in protobuf format. + */ +@InterfaceAudience.Private +public final class FSImageFormatProtobuf { + private static final Log LOG = LogFactory.getLog(FSImageFormatProtobuf.class); + + public static final class Loader implements FSImageFormat.AbstractLoader { + static final int MINIMUM_FILE_LENGTH = 8; + private final Configuration conf; + private final FSNamesystem fsn; + + private String[] stringTable; + + /** The MD5 sum of the loaded file */ + private MD5Hash imgDigest; + /** The transaction ID of the last edit represented by the loaded file */ + private long imgTxId; + + Loader(Configuration conf, FSNamesystem fsn) { + this.conf = conf; + this.fsn = fsn; + } + + @Override + public MD5Hash getLoadedImageMd5() { + return imgDigest; + } + + @Override + public long getLoadedImageTxId() { + return imgTxId; + } + + public String[] getStringTable() { + return stringTable; + } + + void load(File file) throws IOException { + long start = System.currentTimeMillis(); + imgDigest = MD5FileUtils.computeMd5ForFile(file); + RandomAccessFile raFile = new RandomAccessFile(file, "r"); + FileInputStream fin = new FileInputStream(file); + try { + loadInternal(raFile, fin); + long end = System.currentTimeMillis(); + LOG.info("Loaded FSImage in " + (end - start) / 1000 + " seconds."); + } finally { + fin.close(); + raFile.close(); + } + } + + private void loadInternal(RandomAccessFile raFile, FileInputStream fin) + throws IOException { + if (!FSImageUtil.checkFileFormat(raFile)) { + throw new IOException("Unrecognized file format"); + } + FileSummary summary = FSImageUtil.loadSummary(raFile); + + FileChannel channel = fin.getChannel(); + + FSImageFormatPBINode.Loader inodeLoader = new FSImageFormatPBINode.Loader( + fsn, this); + FSImageFormatPBSnapshot.Loader snapshotLoader = new FSImageFormatPBSnapshot.Loader( + fsn, this); + + ArrayList sections = Lists.newArrayList(summary + .getSectionsList()); + Collections.sort(sections, new Comparator() { + @Override + public int compare(FileSummary.Section s1, FileSummary.Section s2) { + SectionName n1 = SectionName.fromString(s1.getName()); + SectionName n2 = SectionName.fromString(s2.getName()); + if (n1 == null) { + return n2 == null ? 0 : -1; + } else if (n2 == null) { + return -1; + } else { + return n1.ordinal() - n2.ordinal(); + } + } + }); + + StartupProgress prog = NameNode.getStartupProgress(); + /** + * beginStep() and the endStep() calls do not match the boundary of the + * sections. This is because that the current implementation only allows + * a particular step to be started for once. + */ + Step currentStep = null; + + for (FileSummary.Section s : sections) { + channel.position(s.getOffset()); + InputStream in = new BufferedInputStream(new LimitInputStream(fin, + s.getLength())); + + in = FSImageUtil.wrapInputStreamForCompression(conf, + summary.getCodec(), in); + + String n = s.getName(); + + switch (SectionName.fromString(n)) { + case NS_INFO: + loadNameSystemSection(in); + break; + case STRING_TABLE: + loadStringTableSection(in); + break; + case INODE: { + currentStep = new Step(StepType.INODES); + prog.beginStep(Phase.LOADING_FSIMAGE, currentStep); + inodeLoader.loadINodeSection(in); + } + break; + case INODE_DIR: + inodeLoader.loadINodeDirectorySection(in); + break; + case FILES_UNDERCONSTRUCTION: + inodeLoader.loadFilesUnderConstructionSection(in); + break; + case SNAPSHOT: + snapshotLoader.loadSnapshotSection(in); + break; + case SNAPSHOT_DIFF: + snapshotLoader.loadSnapshotDiffSection(in); + break; + case SECRET_MANAGER: { + prog.endStep(Phase.LOADING_FSIMAGE, currentStep); + Step step = new Step(StepType.DELEGATION_TOKENS); + prog.beginStep(Phase.LOADING_FSIMAGE, step); + loadSecretManagerSection(in); + prog.endStep(Phase.LOADING_FSIMAGE, step); + } + break; + case CACHE_MANAGER: { + Step step = new Step(StepType.CACHE_POOLS); + prog.beginStep(Phase.LOADING_FSIMAGE, step); + loadCacheManagerSection(in); + prog.endStep(Phase.LOADING_FSIMAGE, step); + } + break; + default: + LOG.warn("Unregconized section " + n); + break; + } + } + } + + private void loadNameSystemSection(InputStream in) throws IOException { + NameSystemSection s = NameSystemSection.parseDelimitedFrom(in); + fsn.setGenerationStampV1(s.getGenstampV1()); + fsn.setGenerationStampV2(s.getGenstampV2()); + fsn.setGenerationStampV1Limit(s.getGenstampV1Limit()); + fsn.setLastAllocatedBlockId(s.getLastAllocatedBlockId()); + imgTxId = s.getTransactionId(); + } + + private void loadStringTableSection(InputStream in) throws IOException { + StringTableSection s = StringTableSection.parseDelimitedFrom(in); + stringTable = new String[s.getNumEntry() + 1]; + for (int i = 0; i < s.getNumEntry(); ++i) { + StringTableSection.Entry e = StringTableSection.Entry + .parseDelimitedFrom(in); + stringTable[e.getId()] = e.getStr(); + } + } + + private void loadSecretManagerSection(InputStream in) throws IOException { + SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(in); + int numKeys = s.getNumKeys(), numTokens = s.getNumTokens(); + ArrayList keys = Lists + .newArrayListWithCapacity(numKeys); + ArrayList tokens = Lists + .newArrayListWithCapacity(numTokens); + + for (int i = 0; i < numKeys; ++i) + keys.add(SecretManagerSection.DelegationKey.parseDelimitedFrom(in)); + + for (int i = 0; i < numTokens; ++i) + tokens.add(SecretManagerSection.PersistToken.parseDelimitedFrom(in)); + + fsn.loadSecretManagerState(s, keys, tokens); + } + + private void loadCacheManagerSection(InputStream in) throws IOException { + CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(in); + ArrayList pools = Lists.newArrayListWithCapacity(s + .getNumPools()); + ArrayList directives = Lists + .newArrayListWithCapacity(s.getNumDirectives()); + for (int i = 0; i < s.getNumPools(); ++i) + pools.add(CachePoolInfoProto.parseDelimitedFrom(in)); + for (int i = 0; i < s.getNumDirectives(); ++i) + directives.add(CacheDirectiveInfoProto.parseDelimitedFrom(in)); + fsn.getCacheManager().loadState( + new CacheManager.PersistState(s, pools, directives)); + } + + } + + public static final class Saver { + private final SaveNamespaceContext context; + private long currentOffset = FSImageUtil.MAGIC_HEADER.length; + private MD5Hash savedDigest; + private StringMap stringMap = new StringMap(); + + private FileChannel fileChannel; + // OutputStream for the section data + private OutputStream sectionOutputStream; + private CompressionCodec codec; + private OutputStream underlyingOutputStream; + public static final int CHECK_CANCEL_INTERVAL = 4096; + + Saver(SaveNamespaceContext context) { + this.context = context; + } + + public MD5Hash getSavedDigest() { + return savedDigest; + } + + public SaveNamespaceContext getContext() { + return context; + } + + public void commitSection(FileSummary.Builder summary, SectionName name) + throws IOException { + long oldOffset = currentOffset; + flushSectionOutputStream(); + + if (codec != null) { + sectionOutputStream = codec.createOutputStream(underlyingOutputStream); + } else { + sectionOutputStream = underlyingOutputStream; + } + long length = fileChannel.position() - oldOffset; + summary.addSections(FileSummary.Section.newBuilder().setName(name.name) + .setLength(length).setOffset(currentOffset)); + currentOffset += length; + } + + private void flushSectionOutputStream() throws IOException { + if (codec != null) { + ((CompressorStream) sectionOutputStream).finish(); + } + sectionOutputStream.flush(); + } + + void save(File file, FSImageCompression compression) throws IOException { + FileOutputStream fout = new FileOutputStream(file); + fileChannel = fout.getChannel(); + try { + saveInternal(fout, compression, file.getAbsolutePath().toString()); + } finally { + fout.close(); + } + } + + private static void saveFileSummary(OutputStream out, FileSummary summary) + throws IOException { + summary.writeDelimitedTo(out); + int length = getOndiskTrunkSize(summary); + byte[] lengthBytes = new byte[4]; + ByteBuffer.wrap(lengthBytes).asIntBuffer().put(length); + out.write(lengthBytes); + } + + private void saveInodes(FileSummary.Builder summary) throws IOException { + FSImageFormatPBINode.Saver saver = new FSImageFormatPBINode.Saver(this, + summary); + + saver.serializeINodeSection(sectionOutputStream); + saver.serializeINodeDirectorySection(sectionOutputStream); + saver.serializeFilesUCSection(sectionOutputStream); + } + + private void saveSnapshots(FileSummary.Builder summary) throws IOException { + FSImageFormatPBSnapshot.Saver snapshotSaver = new FSImageFormatPBSnapshot.Saver( + this, summary, context, context.getSourceNamesystem()); + + snapshotSaver.serializeSnapshotSection(sectionOutputStream); + snapshotSaver.serializeSnapshotDiffSection(sectionOutputStream); + } + + private void saveInternal(FileOutputStream fout, + FSImageCompression compression, String filePath) throws IOException { + StartupProgress prog = NameNode.getStartupProgress(); + MessageDigest digester = MD5Hash.getDigester(); + + underlyingOutputStream = new DigestOutputStream(new BufferedOutputStream( + fout), digester); + underlyingOutputStream.write(FSImageUtil.MAGIC_HEADER); + + fileChannel = fout.getChannel(); + + FileSummary.Builder b = FileSummary.newBuilder() + .setOndiskVersion(FSImageUtil.FILE_VERSION) + .setLayoutVersion(LayoutVersion.getCurrentLayoutVersion()); + + codec = compression.getImageCodec(); + if (codec != null) { + b.setCodec(codec.getClass().getCanonicalName()); + sectionOutputStream = codec.createOutputStream(underlyingOutputStream); + } else { + sectionOutputStream = underlyingOutputStream; + } + + saveNameSystemSection(b); + // Check for cancellation right after serializing the name system section. + // Some unit tests, such as TestSaveNamespace#testCancelSaveNameSpace + // depends on this behavior. + context.checkCancelled(); + + Step step = new Step(StepType.INODES, filePath); + prog.beginStep(Phase.SAVING_CHECKPOINT, step); + saveInodes(b); + saveSnapshots(b); + prog.endStep(Phase.SAVING_CHECKPOINT, step); + + step = new Step(StepType.DELEGATION_TOKENS, filePath); + prog.beginStep(Phase.SAVING_CHECKPOINT, step); + saveSecretManagerSection(b); + prog.endStep(Phase.SAVING_CHECKPOINT, step); + + step = new Step(StepType.CACHE_POOLS, filePath); + prog.beginStep(Phase.SAVING_CHECKPOINT, step); + saveCacheManagerSection(b); + prog.endStep(Phase.SAVING_CHECKPOINT, step); + + saveStringTableSection(b); + + // We use the underlyingOutputStream to write the header. Therefore flush + // the buffered stream (which is potentially compressed) first. + flushSectionOutputStream(); + + FileSummary summary = b.build(); + saveFileSummary(underlyingOutputStream, summary); + underlyingOutputStream.close(); + savedDigest = new MD5Hash(digester.digest()); + } + + private void saveSecretManagerSection(FileSummary.Builder summary) + throws IOException { + final FSNamesystem fsn = context.getSourceNamesystem(); + DelegationTokenSecretManager.SecretManagerState state = fsn + .saveSecretManagerState(); + state.section.writeDelimitedTo(sectionOutputStream); + for (SecretManagerSection.DelegationKey k : state.keys) + k.writeDelimitedTo(sectionOutputStream); + + for (SecretManagerSection.PersistToken t : state.tokens) + t.writeDelimitedTo(sectionOutputStream); + + commitSection(summary, SectionName.SECRET_MANAGER); + } + + private void saveCacheManagerSection(FileSummary.Builder summary) + throws IOException { + final FSNamesystem fsn = context.getSourceNamesystem(); + CacheManager.PersistState state = fsn.getCacheManager().saveState(); + state.section.writeDelimitedTo(sectionOutputStream); + + for (CachePoolInfoProto p : state.pools) + p.writeDelimitedTo(sectionOutputStream); + + for (CacheDirectiveInfoProto p : state.directives) + p.writeDelimitedTo(sectionOutputStream); + + commitSection(summary, SectionName.CACHE_MANAGER); + } + + private void saveNameSystemSection(FileSummary.Builder summary) + throws IOException { + final FSNamesystem fsn = context.getSourceNamesystem(); + OutputStream out = sectionOutputStream; + NameSystemSection.Builder b = NameSystemSection.newBuilder() + .setGenstampV1(fsn.getGenerationStampV1()) + .setGenstampV1Limit(fsn.getGenerationStampV1Limit()) + .setGenstampV2(fsn.getGenerationStampV2()) + .setLastAllocatedBlockId(fsn.getLastAllocatedBlockId()) + .setTransactionId(context.getTxId()); + + // We use the non-locked version of getNamespaceInfo here since + // the coordinating thread of saveNamespace already has read-locked + // the namespace for us. If we attempt to take another readlock + // from the actual saver thread, there's a potential of a + // fairness-related deadlock. See the comments on HDFS-2223. + b.setNamespaceId(fsn.unprotectedGetNamespaceInfo().getNamespaceID()); + NameSystemSection s = b.build(); + s.writeDelimitedTo(out); + + commitSection(summary, SectionName.NS_INFO); + } + + private void saveStringTableSection(FileSummary.Builder summary) + throws IOException { + OutputStream out = sectionOutputStream; + StringTableSection.Builder b = StringTableSection.newBuilder() + .setNumEntry(stringMap.size()); + b.build().writeDelimitedTo(out); + for (Entry e : stringMap.entrySet()) { + StringTableSection.Entry.Builder eb = StringTableSection.Entry + .newBuilder().setId(e.getValue()).setStr(e.getKey()); + eb.build().writeDelimitedTo(out); + } + commitSection(summary, SectionName.STRING_TABLE); + } + + public StringMap getStringMap() { + return stringMap; + } + } + + public static class StringMap { + private final Map stringMap; + + public StringMap() { + stringMap = Maps.newHashMap(); + } + + int getStringId(String str) { + if (str == null) { + return 0; + } + Integer v = stringMap.get(str); + if (v == null) { + int nv = stringMap.size() + 1; + stringMap.put(str, nv); + return nv; + } + return v; + } + + int size() { + return stringMap.size(); + } + + Set> entrySet() { + return stringMap.entrySet(); + } + } + + /** + * Supported section name. The order of the enum determines the order of + * loading. + */ + public enum SectionName { + NS_INFO("NS_INFO"), + STRING_TABLE("STRING_TABLE"), + INODE("INODE"), + SNAPSHOT("SNAPSHOT"), + INODE_DIR("INODE_DIR"), + FILES_UNDERCONSTRUCTION("FILES_UNDERCONSTRUCTION"), + SNAPSHOT_DIFF("SNAPSHOT_DIFF"), + SECRET_MANAGER("SECRET_MANAGER"), + CACHE_MANAGER("CACHE_MANAGER"); + + private static final SectionName[] values = SectionName.values(); + + public static SectionName fromString(String name) { + for (SectionName n : values) { + if (n.name.equals(name)) + return n; + } + return null; + } + + private final String name; + + private SectionName(String name) { + this.name = name; + } + } + + private static int getOndiskTrunkSize(com.google.protobuf.GeneratedMessage s) { + return CodedOutputStream.computeRawVarint32Size(s.getSerializedSize()) + + s.getSerializedSize(); + } + + private FSImageFormatProtobuf() { + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java new file mode 100644 index 00000000000..b9953480f26 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.util.Arrays; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.Loader; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; +import org.apache.hadoop.io.compress.CompressionCodec; + +@InterfaceAudience.Private +public final class FSImageUtil { + public static final byte[] MAGIC_HEADER = "HDFSIMG1".getBytes(); + public static final int FILE_VERSION = 1; + + public static boolean checkFileFormat(RandomAccessFile file) + throws IOException { + if (file.length() < Loader.MINIMUM_FILE_LENGTH) + return false; + + byte[] magic = new byte[MAGIC_HEADER.length]; + file.readFully(magic); + if (!Arrays.equals(MAGIC_HEADER, magic)) + return false; + + return true; + } + + public static FileSummary loadSummary(RandomAccessFile file) + throws IOException { + final int FILE_LENGTH_FIELD_SIZE = 4; + long fileLength = file.length(); + file.seek(fileLength - FILE_LENGTH_FIELD_SIZE); + int summaryLength = file.readInt(); + + if (summaryLength <= 0) { + throw new IOException("Negative length of the file"); + } + file.seek(fileLength - FILE_LENGTH_FIELD_SIZE - summaryLength); + + byte[] summaryBytes = new byte[summaryLength]; + file.readFully(summaryBytes); + + FileSummary summary = FileSummary + .parseDelimitedFrom(new ByteArrayInputStream(summaryBytes)); + if (summary.getOndiskVersion() != FILE_VERSION) { + throw new IOException("Unsupported file version " + + summary.getOndiskVersion()); + } + + if (!LayoutVersion.supports(Feature.PROTOBUF_FORMAT, + summary.getLayoutVersion())) { + throw new IOException("Unsupported layout version " + + summary.getLayoutVersion()); + } + return summary; + } + + public static InputStream wrapInputStreamForCompression( + Configuration conf, String codec, InputStream in) throws IOException { + if (codec.isEmpty()) + return in; + + FSImageCompression compression = FSImageCompression.createCompression( + conf, codec); + CompressionCodec imageCodec = compression.getImageCodec(); + return imageCodec.createInputStream(in); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 4a75746e872..eaa363e0a0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -180,6 +180,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState; import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; @@ -197,6 +198,8 @@ import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.Util; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection.PersistToken; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; @@ -5992,6 +5995,15 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } + /** + * @return all the under-construction files in the lease map + */ + Map getFilesUnderConstruction() { + synchronized (leaseManager) { + return leaseManager.getINodesUnderConstruction(); + } + } + /** * Register a Backup name-node, verifying that it belongs * to the correct namespace, and adding it to the set of @@ -6268,6 +6280,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats, dtSecretManager.saveSecretManagerStateCompat(out, sdPath); } + SecretManagerState saveSecretManagerState() { + return dtSecretManager.saveSecretManagerState(); + } + /** * @param in load the state of secret manager from input stream */ @@ -6275,6 +6291,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, dtSecretManager.loadSecretManagerStateCompat(in); } + void loadSecretManagerState(SecretManagerSection s, + List keys, + List tokens) throws IOException { + dtSecretManager.loadSecretManagerState(new SecretManagerState(s, keys, tokens)); + } + /** * Log the updateMasterKey operation to edit logs * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 83cb0a4eb94..f9a06f1e5bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -171,7 +171,7 @@ public class INodeDirectory extends INodeWithAdditionalFields return children == null? -1: Collections.binarySearch(children, name); } - protected DirectoryWithSnapshotFeature addSnapshotFeature( + public DirectoryWithSnapshotFeature addSnapshotFeature( DirectoryDiffList diffs) { Preconditions.checkState(!isWithSnapshot(), "Directory is already with snapshot"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 500405e09d4..80abb5268dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -252,7 +252,7 @@ public class INodeFile extends INodeWithAdditionalFields /* Start of Snapshot Feature */ - private FileWithSnapshotFeature addSnapshotFeature(FileDiffList diffs) { + public FileWithSnapshotFeature addSnapshotFeature(FileDiffList diffs) { Preconditions.checkState(!isWithSnapshot(), "File is already with snapshot"); FileWithSnapshotFeature sf = new FileWithSnapshotFeature(diffs); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java index 5ffcc21f5bb..bd0355b6618 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.util.Iterator; import java.util.List; import org.apache.hadoop.fs.permission.FsPermission; @@ -46,6 +47,10 @@ public class INodeMap { /** Synchronized by external lock. */ private final GSet map; + public Iterator getMapIterator() { + return map.iterator(); + } + private INodeMap(GSet map) { Preconditions.checkArgument(map != null); this.map = map; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java index 67ee88e11de..a7c4c75f005 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.util.Canceler; @@ -32,7 +33,8 @@ import com.google.common.base.Preconditions; * allows cancellation, and also is responsible for accumulating * failed storage directories. */ -class SaveNamespaceContext { +@InterfaceAudience.Private +public class SaveNamespaceContext { private final FSNamesystem sourceNamesystem; private final long txid; private final List errorSDs = @@ -72,7 +74,7 @@ class SaveNamespaceContext { completionLatch.countDown(); } - void checkCancelled() throws SaveNamespaceCancelledException { + public void checkCancelled() throws SaveNamespaceCancelledException { if (canceller.isCancelled()) { throw new SaveNamespaceCancelledException( canceller.getCancellationReason()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index 06f7a89e33a..a9cad94f0b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -244,7 +244,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { this.isSnapshotRoot = isSnapshotRoot; } - ChildrenDiff getChildrenDiff() { + public ChildrenDiff getChildrenDiff() { return diff; } @@ -343,6 +343,10 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { return super.toString() + " childrenSize=" + childrenSize + ", " + diff; } + int getChildrenSize() { + return childrenSize; + } + @Override void write(DataOutput out, ReferenceMap referenceMap) throws IOException { writeSnapshot(out); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java new file mode 100644 index 00000000000..06cc1d0ac1f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java @@ -0,0 +1,437 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Loader.loadINodeDirectory; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Loader.loadINodeReference; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Loader.loadPermission; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Loader.updateBlocksMap; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Saver.buildINodeDirectory; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Saver.buildINodeFile; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Saver.buildINodeReference; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection.CreatedListEntry; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection.DiffEntry.Type; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes; +import org.apache.hadoop.hdfs.server.namenode.INodeMap; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields; +import org.apache.hadoop.hdfs.server.namenode.SaveNamespaceContext; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root; +import org.apache.hadoop.hdfs.util.Diff.ListType; + +import com.google.common.base.Preconditions; +import com.google.protobuf.ByteString; + +@InterfaceAudience.Private +public class FSImageFormatPBSnapshot { + /** + * Loading snapshot related information from protobuf based FSImage + */ + public final static class Loader { + private final FSNamesystem fsn; + private final FSDirectory fsDir; + private final FSImageFormatProtobuf.Loader parent; + private final Map snapshotMap; + + + public Loader(FSNamesystem fsn, FSImageFormatProtobuf.Loader parent) { + this.fsn = fsn; + this.fsDir = fsn.getFSDirectory(); + this.snapshotMap = new HashMap(); + this.parent = parent; + } + + /** + * Load the snapshots section from fsimage. Also convert snapshottable + * directories into {@link INodeDirectorySnapshottable}. + * + */ + public void loadSnapshotSection(InputStream in) throws IOException { + SnapshotManager sm = fsn.getSnapshotManager(); + SnapshotSection section = SnapshotSection.parseDelimitedFrom(in); + int snum = section.getNumSnapshots(); + sm.setNumSnapshots(snum); + sm.setSnapshotCounter(section.getSnapshotCounter()); + for (long sdirId : section.getSnapshottableDirList()) { + INodeDirectory dir = fsDir.getInode(sdirId).asDirectory(); + final INodeDirectorySnapshottable sdir; + if (!dir.isSnapshottable()) { + sdir = new INodeDirectorySnapshottable(dir); + fsDir.addToInodeMap(sdir); + } else { + // dir is root, and admin set root to snapshottable before + sdir = (INodeDirectorySnapshottable) dir; + sdir.setSnapshotQuota(INodeDirectorySnapshottable.SNAPSHOT_LIMIT); + } + sm.addSnapshottable(sdir); + } + loadSnapshots(in, snum); + } + + private void loadSnapshots(InputStream in, int size) throws IOException { + for (int i = 0; i < size; i++) { + SnapshotSection.Snapshot pbs = SnapshotSection.Snapshot + .parseDelimitedFrom(in); + INodeDirectory root = loadINodeDirectory(pbs.getRoot(), + parent.getStringTable()); + int sid = pbs.getSnapshotId(); + INodeDirectorySnapshottable parent = (INodeDirectorySnapshottable) fsDir + .getInode(root.getId()).asDirectory(); + Snapshot snapshot = new Snapshot(sid, root, parent); + // add the snapshot to parent, since we follow the sequence of + // snapshotsByNames when saving, we do not need to sort when loading + parent.addSnapshot(snapshot); + snapshotMap.put(sid, snapshot); + } + } + + /** + * Load the snapshot diff section from fsimage. + */ + public void loadSnapshotDiffSection(InputStream in) throws IOException { + while (true) { + SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry + .parseDelimitedFrom(in); + if (entry == null) { + break; + } + long inodeId = entry.getInodeId(); + INode inode = fsDir.getInode(inodeId); + SnapshotDiffSection.DiffEntry.Type type = entry.getType(); + switch (type) { + case FILEDIFF: + loadFileDiffList(in, inode.asFile(), entry.getNumOfDiff()); + break; + case DIRECTORYDIFF: + loadDirectoryDiffList(in, inode.asDirectory(), entry.getNumOfDiff()); + break; + } + } + } + + /** Load FileDiff list for a file with snapshot feature */ + private void loadFileDiffList(InputStream in, INodeFile file, int size) + throws IOException { + final FileDiffList diffs = new FileDiffList(); + for (int i = 0; i < size; i++) { + SnapshotDiffSection.FileDiff pbf = SnapshotDiffSection.FileDiff + .parseDelimitedFrom(in); + INodeFileAttributes copy = null; + if (pbf.hasSnapshotCopy()) { + INodeSection.INodeFile fileInPb = pbf.getSnapshotCopy(); + PermissionStatus permission = loadPermission( + fileInPb.getPermission(), parent.getStringTable()); + copy = new INodeFileAttributes.SnapshotCopy(pbf.getName() + .toByteArray(), permission, fileInPb.getModificationTime(), + fileInPb.getAccessTime(), (short) fileInPb.getReplication(), + fileInPb.getPreferredBlockSize()); + } + + FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null, + pbf.getFileSize()); + diffs.addFirst(diff); + } + file.addSnapshotFeature(diffs); + } + + /** Load the created list in a DirectoryDiff */ + private List loadCreatedList(InputStream in, INodeDirectory dir, + int size) throws IOException { + List clist = new ArrayList(size); + for (long c = 0; c < size; c++) { + CreatedListEntry entry = CreatedListEntry.parseDelimitedFrom(in); + INode created = SnapshotFSImageFormat.loadCreated(entry.getName() + .toByteArray(), dir); + clist.add(created); + } + return clist; + } + + private void addToDeletedList(INode dnode, INodeDirectory parent) { + dnode.setParent(parent); + if (dnode.isFile()) { + updateBlocksMap(dnode.asFile(), fsn.getBlockManager()); + } + } + + /** + * Load the deleted list in a DirectoryDiff + * @param totalSize the total size of the deleted list + * @param deletedNodes non-reference inodes in the deleted list. These + * inodes' ids are directly recorded in protobuf + */ + private List loadDeletedList(InputStream in, INodeDirectory dir, + int refNum, List deletedNodes) throws IOException { + List dlist = new ArrayList(refNum + deletedNodes.size()); + // load non-reference inodes + for (long deletedId : deletedNodes) { + INode deleted = fsDir.getInode(deletedId); + dlist.add(deleted); + addToDeletedList(deleted, dir); + } + // load reference nodes in the deleted list + for (int r = 0; r < refNum; r++) { + INodeSection.INodeReference ref = INodeSection.INodeReference + .parseDelimitedFrom(in); + INodeReference refNode = loadINodeReference(ref, fsDir); + dlist.add(refNode); + addToDeletedList(refNode, dir); + } + Collections.sort(dlist, new Comparator() { + @Override + public int compare(INode n1, INode n2) { + return n1.compareTo(n2.getLocalNameBytes()); + } + }); + return dlist; + } + + /** Load DirectoryDiff list for a directory with snapshot feature */ + private void loadDirectoryDiffList(InputStream in, INodeDirectory dir, + int size) throws IOException { + if (!dir.isWithSnapshot()) { + dir.addSnapshotFeature(null); + } + DirectoryDiffList diffs = dir.getDiffs(); + for (int i = 0; i < size; i++) { + // load a directory diff + SnapshotDiffSection.DirectoryDiff diffInPb = SnapshotDiffSection. + DirectoryDiff.parseDelimitedFrom(in); + final int snapshotId = diffInPb.getSnapshotId(); + final Snapshot snapshot = snapshotMap.get(snapshotId); + int childrenSize = diffInPb.getChildrenSize(); + boolean useRoot = diffInPb.getIsSnapshotRoot(); + INodeDirectoryAttributes copy = null; + if (useRoot) { + copy = snapshot.getRoot(); + }else if (diffInPb.hasSnapshotCopy()) { + INodeSection.INodeDirectory dirCopyInPb = diffInPb.getSnapshotCopy(); + final byte[] name = diffInPb.getName().toByteArray(); + PermissionStatus permission = loadPermission(dirCopyInPb + .getPermission(), parent.getStringTable()); + long modTime = dirCopyInPb.getModificationTime(); + boolean noQuota = dirCopyInPb.getNsQuota() == -1 + && dirCopyInPb.getDsQuota() == -1; + copy = noQuota ? new INodeDirectoryAttributes.SnapshotCopy(name, + permission, modTime) + : new INodeDirectoryAttributes.CopyWithQuota(name, permission, + modTime, dirCopyInPb.getNsQuota(), dirCopyInPb.getDsQuota()); + } + // load created list + List clist = loadCreatedList(in, dir, + diffInPb.getCreatedListSize()); + // load deleted list + List dlist = loadDeletedList(in, dir, + diffInPb.getNumOfDeletedRef(), diffInPb.getDeletedINodeList()); + // create the directory diff + DirectoryDiff diff = new DirectoryDiff(snapshotId, copy, null, + childrenSize, clist, dlist, useRoot); + diffs.addFirst(diff); + } + } + } + + /** + * Saving snapshot related information to protobuf based FSImage + */ + public final static class Saver { + private final FSNamesystem fsn; + private final FileSummary.Builder headers; + private final FSImageFormatProtobuf.Saver parent; + private final SaveNamespaceContext context; + + public Saver(FSImageFormatProtobuf.Saver parent, + FileSummary.Builder headers, SaveNamespaceContext context, FSNamesystem fsn) { + this.parent = parent; + this.headers = headers; + this.context = context; + this.fsn = fsn; + } + + /** + * save all the snapshottable directories and snapshots to fsimage + */ + public void serializeSnapshotSection(OutputStream out) throws IOException { + SnapshotManager sm = fsn.getSnapshotManager(); + SnapshotSection.Builder b = SnapshotSection.newBuilder() + .setSnapshotCounter(sm.getSnapshotCounter()) + .setNumSnapshots(sm.getNumSnapshots()); + + INodeDirectorySnapshottable[] snapshottables = sm.getSnapshottableDirs(); + for (INodeDirectorySnapshottable sdir : snapshottables) { + b.addSnapshottableDir(sdir.getId()); + } + b.build().writeDelimitedTo(out); + int i = 0; + for(INodeDirectorySnapshottable sdir : snapshottables) { + for(Snapshot s : sdir.getSnapshotsByNames()) { + Root sroot = s.getRoot(); + SnapshotSection.Snapshot.Builder sb = SnapshotSection.Snapshot + .newBuilder().setSnapshotId(s.getId()); + INodeSection.INodeDirectory.Builder db = buildINodeDirectory(sroot, + parent.getStringMap()); + INodeSection.INode r = INodeSection.INode.newBuilder() + .setId(sroot.getId()) + .setType(INodeSection.INode.Type.DIRECTORY) + .setName(ByteString.copyFrom(sroot.getLocalNameBytes())) + .setDirectory(db).build(); + sb.setRoot(r).build().writeDelimitedTo(out); + i++; + if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { + context.checkCancelled(); + } + } + } + Preconditions.checkState(i == sm.getNumSnapshots()); + parent.commitSection(headers, FSImageFormatProtobuf.SectionName.SNAPSHOT); + } + + /** + * save all the snapshot diff to fsimage + */ + public void serializeSnapshotDiffSection(OutputStream out) + throws IOException { + INodeMap inodesMap = fsn.getFSDirectory().getINodeMap(); + int i = 0; + Iterator iter = inodesMap.getMapIterator(); + while (iter.hasNext()) { + INodeWithAdditionalFields inode = iter.next(); + if (inode.isFile()) { + serializeFileDiffList(inode.asFile(), out); + } else if (inode.isDirectory()) { + serializeDirDiffList(inode.asDirectory(), out); + } + ++i; + if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { + context.checkCancelled(); + } + } + parent.commitSection(headers, + FSImageFormatProtobuf.SectionName.SNAPSHOT_DIFF); + } + + private void serializeFileDiffList(INodeFile file, OutputStream out) + throws IOException { + FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature(); + if (sf != null) { + List diffList = sf.getDiffs().asList(); + SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry + .newBuilder().setInodeId(file.getId()).setType(Type.FILEDIFF) + .setNumOfDiff(diffList.size()).build(); + entry.writeDelimitedTo(out); + for (int i = diffList.size() - 1; i >= 0; i--) { + FileDiff diff = diffList.get(i); + SnapshotDiffSection.FileDiff.Builder fb = SnapshotDiffSection.FileDiff + .newBuilder().setSnapshotId(diff.getSnapshotId()) + .setFileSize(diff.getFileSize()); + INodeFileAttributes copy = diff.snapshotINode; + if (copy != null) { + fb.setName(ByteString.copyFrom(copy.getLocalNameBytes())) + .setSnapshotCopy(buildINodeFile(copy, parent.getStringMap())); + } + fb.build().writeDelimitedTo(out); + } + } + } + + private void saveCreatedDeletedList(List created, + List deletedRefs, OutputStream out) throws IOException { + // local names of the created list member + for (INode c : created) { + SnapshotDiffSection.CreatedListEntry.newBuilder() + .setName(ByteString.copyFrom(c.getLocalNameBytes())).build() + .writeDelimitedTo(out); + } + // reference nodes in deleted list + for (INodeReference ref : deletedRefs) { + INodeSection.INodeReference.Builder rb = buildINodeReference(ref); + rb.build().writeDelimitedTo(out); + } + } + + private void serializeDirDiffList(INodeDirectory dir, OutputStream out) + throws IOException { + DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature(); + if (sf != null) { + List diffList = sf.getDiffs().asList(); + SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry + .newBuilder().setInodeId(dir.getId()).setType(Type.DIRECTORYDIFF) + .setNumOfDiff(diffList.size()).build(); + entry.writeDelimitedTo(out); + for (int i = diffList.size() - 1; i >= 0; i--) { // reverse order! + DirectoryDiff diff = diffList.get(i); + SnapshotDiffSection.DirectoryDiff.Builder db = SnapshotDiffSection. + DirectoryDiff.newBuilder().setSnapshotId(diff.getSnapshotId()) + .setChildrenSize(diff.getChildrenSize()) + .setIsSnapshotRoot(diff.isSnapshotRoot()); + INodeDirectoryAttributes copy = diff.snapshotINode; + if (!diff.isSnapshotRoot() && copy != null) { + db.setName(ByteString.copyFrom(copy.getLocalNameBytes())) + .setSnapshotCopy( + buildINodeDirectory(copy, parent.getStringMap())); + } + // process created list and deleted list + List created = diff.getChildrenDiff() + .getList(ListType.CREATED); + db.setCreatedListSize(created.size()); + List deleted = diff.getChildrenDiff().getList(ListType.DELETED); + List refs = new ArrayList(); + for (INode d : deleted) { + if (d.isReference()) { + refs.add(d.asReference()); + } else { + db.addDeletedINode(d.getId()); + } + } + db.setNumOfDeletedRef(refs.size()); + db.build().writeDelimitedTo(out); + saveCreatedDeletedList(created, refs, out); + } + } + } + } + + private FSImageFormatPBSnapshot(){} +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java index e836cd87959..69fdf97391c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java @@ -27,7 +27,6 @@ import java.util.Map; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; -import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; @@ -137,7 +136,7 @@ public class SnapshotFSImageFormat { * @param parent The directory that the created list belongs to. * @return The created node. */ - private static INode loadCreated(byte[] createdNodeName, + public static INode loadCreated(byte[] createdNodeName, INodeDirectory parent) throws IOException { // the INode in the created list should be a reference to another INode // in posterior SnapshotDiffs or one of the current children @@ -209,11 +208,13 @@ public class SnapshotFSImageFormat { /** * Load snapshots and snapshotQuota for a Snapshottable directory. - * @param snapshottableParent The snapshottable directory for loading. - * @param numSnapshots The number of snapshots that the directory has. - * @param in The {@link DataInput} instance to read. - * @param loader The {@link Loader} instance that this loading procedure is - * using. + * + * @param snapshottableParent + * The snapshottable directory for loading. + * @param numSnapshots + * The number of snapshots that the directory has. + * @param loader + * The loader */ public static void loadSnapshotList( INodeDirectorySnapshottable snapshottableParent, int numSnapshots, @@ -231,10 +232,13 @@ public class SnapshotFSImageFormat { /** * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot * directory. - * @param dir The snapshottable directory for loading. - * @param in The {@link DataInput} instance to read. - * @param loader The {@link Loader} instance that this loading procedure is - * using. + * + * @param dir + * The snapshottable directory for loading. + * @param in + * The {@link DataInput} instance to read. + * @param loader + * The loader */ public static void loadDirectoryDiffList(INodeDirectory dir, DataInput in, FSImageFormat.Loader loader) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 8fa0f0c932b..be1ddc0e9e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -270,6 +270,23 @@ public class SnapshotManager implements SnapshotStats { return numSnapshots.get(); } + void setNumSnapshots(int num) { + numSnapshots.set(num); + } + + int getSnapshotCounter() { + return snapshotCounter; + } + + void setSnapshotCounter(int counter) { + snapshotCounter = counter; + } + + INodeDirectorySnapshottable[] getSnapshottableDirs() { + return snapshottables.values().toArray( + new INodeDirectorySnapshottable[snapshottables.size()]); + } + /** * Write {@link #snapshotCounter}, {@link #numSnapshots}, * and all snapshots to the DataOutput. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java new file mode 100644 index 00000000000..2433b28a859 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; +import org.apache.hadoop.hdfs.server.namenode.FSImageUtil; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; +import org.apache.hadoop.io.IOUtils; + +import com.google.common.base.Preconditions; +import com.google.common.io.LimitInputStream; + +/** + * This is the tool for analyzing file sizes in the namespace image. In order to + * run the tool one should define a range of integers [0, maxSize] by + * specifying maxSize and a step. The range of integers is + * divided into segments of size step: + * [0, s1, ..., sn-1, maxSize], and the visitor + * calculates how many files in the system fall into each segment + * [si-1, si). Note that files larger than + * maxSize always fall into the very last segment. + * + *

Input.

+ *
    + *
  • filename specifies the location of the image file;
    • + *
  • maxSize determines the range [0, maxSize] of files + * sizes considered by the visitor;
    • + *
  • step the range is divided into segments of size step.
    • + *
+ * + *

Output.

The output file is formatted as a tab separated two column + * table: Size and NumFiles. Where Size represents the start of the segment, and + * numFiles is the number of files form the image which size falls in this + * segment. + * + */ +final class FileDistributionCalculator { + private final static long MAX_SIZE_DEFAULT = 0x2000000000L; // 1/8 TB = 2^37 + private final static int INTERVAL_DEFAULT = 0x200000; // 2 MB = 2^21 + + private final Configuration conf; + private final long maxSize; + private final int steps; + private final PrintWriter out; + + private int[] distribution; + private int totalFiles; + private int totalDirectories; + private int totalBlocks; + private long totalSpace; + private long maxFileSize; + + FileDistributionCalculator(Configuration conf, long maxSize, int steps, + PrintWriter out) { + this.conf = conf; + this.maxSize = maxSize == 0 ? MAX_SIZE_DEFAULT : maxSize; + this.steps = steps == 0 ? INTERVAL_DEFAULT : steps; + this.out = out; + long numIntervals = this.maxSize / this.steps; + this.distribution = new int[1 + (int) (numIntervals)]; + Preconditions.checkState(numIntervals < Integer.MAX_VALUE, + "Too many distribution intervals"); + } + + void visit(RandomAccessFile file) throws IOException { + if (!FSImageUtil.checkFileFormat(file)) { + throw new IOException("Unrecognized FSImage"); + } + + FileSummary summary = FSImageUtil.loadSummary(file); + FileInputStream in = null; + try { + in = new FileInputStream(file.getFD()); + for (FileSummary.Section s : summary.getSectionsList()) { + if (SectionName.fromString(s.getName()) != SectionName.INODE) { + continue; + } + + in.getChannel().position(s.getOffset()); + InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, + summary.getCodec(), new BufferedInputStream(new LimitInputStream( + in, s.getLength()))); + run(is); + output(); + } + } finally { + IOUtils.cleanup(null, in); + } + } + + private void run(InputStream in) throws IOException { + INodeSection s = INodeSection.parseDelimitedFrom(in); + for (int i = 0; i < s.getNumInodes(); ++i) { + INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); + if (p.getType() == INodeSection.INode.Type.FILE) { + ++totalFiles; + INodeSection.INodeFile f = p.getFile(); + totalBlocks += f.getBlocksCount(); + long fileSize = 0; + for (BlockProto b : f.getBlocksList()) { + fileSize += b.getNumBytes() * f.getReplication(); + } + maxFileSize = Math.max(fileSize, maxFileSize); + totalSpace += fileSize; + + int bucket = fileSize > maxSize ? distribution.length - 1 : (int) Math + .ceil((double)fileSize / steps); + ++distribution[bucket]; + + } else if (p.getType() == INodeSection.INode.Type.DIRECTORY) { + ++totalDirectories; + } + + if (i % (1 << 20) == 0) { + out.println("Processed " + i + " inodes."); + } + } + } + + private void output() { + // write the distribution into the output file + out.print("Size\tNumFiles\n"); + for (int i = 0; i < distribution.length; i++) { + if (distribution[i] != 0) { + out.print(((long) i * steps) + "\t" + distribution[i]); + out.print('\n'); + } + } + out.print("totalFiles = " + totalFiles + "\n"); + out.print("totalDirectories = " + totalDirectories + "\n"); + out.print("totalBlocks = " + totalBlocks + "\n"); + out.print("totalSpace = " + totalSpace + "\n"); + out.print("maxFileSize = " + maxFileSize + "\n"); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java index c529fb5cdc2..19b859118ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java @@ -127,7 +127,7 @@ class ImageLoaderCurrent implements ImageLoader { new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39, - -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50, -51 }; + -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50, -51, -52 }; private int imageVersion = 0; private final Map subtreeMap = new HashMap(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java new file mode 100644 index 00000000000..13c25e965b6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsrPBImage.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; +import org.apache.hadoop.hdfs.server.namenode.FSImageUtil; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeSymlink; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; +import org.apache.hadoop.hdfs.server.namenode.INodeId; +import org.apache.hadoop.io.IOUtils; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.io.LimitInputStream; + +/** + * This is the tool for analyzing file sizes in the namespace image. In order to + * run the tool one should define a range of integers [0, maxSize] by + * specifying maxSize and a step. The range of integers is + * divided into segments of size step: + * [0, s1, ..., sn-1, maxSize], and the visitor + * calculates how many files in the system fall into each segment + * [si-1, si). Note that files larger than + * maxSize always fall into the very last segment. + * + *

Input.

+ *
    + *
  • filename specifies the location of the image file;
    • + *
  • maxSize determines the range [0, maxSize] of files + * sizes considered by the visitor;
    • + *
  • step the range is divided into segments of size step.
    • + *
+ * + *

Output.

The output file is formatted as a tab separated two column + * table: Size and NumFiles. Where Size represents the start of the segment, and + * numFiles is the number of files form the image which size falls in this + * segment. + * + */ +final class LsrPBImage { + private final Configuration conf; + private final PrintWriter out; + private String[] stringTable; + private HashMap inodes = Maps.newHashMap(); + private HashMap dirmap = Maps.newHashMap(); + + public LsrPBImage(Configuration conf, PrintWriter out) { + this.conf = conf; + this.out = out; + } + + public void visit(RandomAccessFile file) throws IOException { + if (!FSImageUtil.checkFileFormat(file)) { + throw new IOException("Unrecognized FSImage"); + } + + FileSummary summary = FSImageUtil.loadSummary(file); + FileInputStream fin = null; + try { + fin = new FileInputStream(file.getFD()); + + ArrayList sections = Lists.newArrayList(summary + .getSectionsList()); + Collections.sort(sections, new Comparator() { + @Override + public int compare(FileSummary.Section s1, FileSummary.Section s2) { + SectionName n1 = SectionName.fromString(s1.getName()); + SectionName n2 = SectionName.fromString(s2.getName()); + if (n1 == null) { + return n2 == null ? 0 : -1; + } else if (n2 == null) { + return -1; + } else { + return n1.ordinal() - n2.ordinal(); + } + } + }); + + for (FileSummary.Section s : sections) { + fin.getChannel().position(s.getOffset()); + InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, + summary.getCodec(), new BufferedInputStream(new LimitInputStream( + fin, s.getLength()))); + + switch (SectionName.fromString(s.getName())) { + case STRING_TABLE: + loadStringTable(is); + break; + case INODE: + loadINodeSection(is); + break; + case INODE_DIR: + loadINodeDirectorySection(is); + break; + default: + break; + } + } + list("", INodeId.ROOT_INODE_ID); + } finally { + IOUtils.cleanup(null, fin); + } + } + + private void list(String parent, long dirId) { + INode inode = inodes.get(dirId); + listINode(parent.isEmpty() ? "/" : parent, inode); + long[] children = dirmap.get(dirId); + if (children == null) { + return; + } + String newParent = parent + inode.getName().toStringUtf8() + "/"; + for (long cid : children) { + list(newParent, cid); + } + } + + private void listINode(String parent, INode inode) { + switch (inode.getType()) { + case FILE: { + INodeFile f = inode.getFile(); + PermissionStatus p = FSImageFormatPBINode.Loader.loadPermission( + f.getPermission(), stringTable); + out.print(String.format("-%s %2s %8s %10s %10s %10d %s%s\n", p + .getPermission().toString(), f.getReplication(), p.getUserName(), p + .getGroupName(), f.getModificationTime(), getFileSize(f), parent, + inode.getName().toStringUtf8())); + } + break; + case DIRECTORY: { + INodeDirectory d = inode.getDirectory(); + PermissionStatus p = FSImageFormatPBINode.Loader.loadPermission( + d.getPermission(), stringTable); + out.print(String.format("d%s - %8s %10s %10s %10d %s%s\n", p + .getPermission().toString(), p.getUserName(), p.getGroupName(), d + .getModificationTime(), 0, parent, inode.getName().toStringUtf8())); + } + break; + case SYMLINK: { + INodeSymlink d = inode.getSymlink(); + PermissionStatus p = FSImageFormatPBINode.Loader.loadPermission( + d.getPermission(), stringTable); + out.print(String.format("-%s - %8s %10s %10s %10d %s%s -> %s\n", p + .getPermission().toString(), p.getUserName(), p.getGroupName(), 0, 0, + parent, inode.getName().toStringUtf8(), d.getTarget().toStringUtf8())); + } + break; + default: + break; + } + } + + private long getFileSize(INodeFile f) { + long size = 0; + for (BlockProto p : f.getBlocksList()) { + size += p.getNumBytes(); + } + return size; + } + + private void loadINodeDirectorySection(InputStream in) throws IOException { + while (true) { + INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry + .parseDelimitedFrom(in); + // note that in is a LimitedInputStream + if (e == null) { + break; + } + long[] l = new long[e.getChildrenCount()]; + for (int i = 0; i < l.length; ++i) { + l[i] = e.getChildren(i); + } + dirmap.put(e.getParent(), l); + for (int i = 0; i < e.getNumOfRef(); i++) { + INodeSection.INodeReference.parseDelimitedFrom(in); + } + } + } + + private void loadINodeSection(InputStream in) throws IOException { + INodeSection s = INodeSection.parseDelimitedFrom(in); + for (int i = 0; i < s.getNumInodes(); ++i) { + INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); + inodes.put(p.getId(), p); + } + } + + private void loadStringTable(InputStream in) throws IOException { + StringTableSection s = StringTableSection.parseDelimitedFrom(in); + stringTable = new String[s.getNumEntry() + 1]; + for (int i = 0; i < s.getNumEntry(); ++i) { + StringTableSection.Entry e = StringTableSection.Entry + .parseDelimitedFrom(in); + stringTable[e.getId()] = e.getStr(); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java new file mode 100644 index 00000000000..d4732af5bd7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.RandomAccessFile; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; + +/** + * OfflineImageViewer to dump the contents of an Hadoop image file to XML or the + * console. Main entry point into utility, either via the command line or + * programatically. + */ +@InterfaceAudience.Private +public class OfflineImageViewerPB { + public static final Log LOG = LogFactory.getLog(OfflineImageViewerPB.class); + + private final static String usage = "Usage: bin/hdfs oiv [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" + + "Offline Image Viewer\n" + + "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n" + + "saving the results in OUTPUTFILE.\n" + + "\n" + + "The oiv utility will attempt to parse correctly formed image files\n" + + "and will abort fail with mal-formed image files.\n" + + "\n" + + "The tool works offline and does not require a running cluster in\n" + + "order to process an image file.\n" + + "\n" + + "The following image processors are available:\n" + + " * Ls: The default image processor generates an lsr-style listing\n" + + " of the files in the namespace, with the same fields in the same\n" + + " order. Note that in order to correctly determine file sizes,\n" + + " this formatter cannot skip blocks and will override the\n" + + " -skipBlocks option.\n" + + " * XML: This processor creates an XML document with all elements of\n" + + " the fsimage enumerated, suitable for further analysis by XML\n" + + " tools.\n" + + " * FileDistribution: This processor analyzes the file size\n" + + " distribution in the image.\n" + + " -maxSize specifies the range [0, maxSize] of file sizes to be\n" + + " analyzed (128GB by default).\n" + + " -step defines the granularity of the distribution. (2MB by default)\n" + + "\n" + + "Required command line arguments:\n" + + "-i,--inputFile FSImage file to process.\n" + + "-o,--outputFile Name of output file. If the specified\n" + + " file exists, it will be overwritten.\n" + + "\n" + + "Optional command line arguments:\n" + + "-p,--processor Select which type of processor to apply\n" + + " against image file." + + " (Ls|XML|FileDistribution).\n" + + "-h,--help Display usage information and exit\n"; + + /** + * Build command-line options and descriptions + */ + private static Options buildOptions() { + Options options = new Options(); + + // Build in/output file arguments, which are required, but there is no + // addOption method that can specify this + OptionBuilder.isRequired(); + OptionBuilder.hasArgs(); + OptionBuilder.withLongOpt("outputFile"); + options.addOption(OptionBuilder.create("o")); + + OptionBuilder.isRequired(); + OptionBuilder.hasArgs(); + OptionBuilder.withLongOpt("inputFile"); + options.addOption(OptionBuilder.create("i")); + + options.addOption("p", "processor", true, ""); + options.addOption("h", "help", false, ""); + options.addOption("skipBlocks", false, ""); + options.addOption("printToScreen", false, ""); + options.addOption("delimiter", true, ""); + + return options; + } + + /** + * Entry point to command-line-driven operation. User may specify options and + * start fsimage viewer from the command line. Program will process image file + * and exit cleanly or, if an error is encountered, inform user and exit. + * + * @param args + * Command line options + * @throws IOException + */ + public static void main(String[] args) throws IOException { + Options options = buildOptions(); + if (args.length == 0) { + printUsage(); + return; + } + + CommandLineParser parser = new PosixParser(); + CommandLine cmd; + + try { + cmd = parser.parse(options, args); + } catch (ParseException e) { + System.out.println("Error parsing command-line options: "); + printUsage(); + return; + } + + if (cmd.hasOption("h")) { // print help and exit + printUsage(); + return; + } + + String inputFile = cmd.getOptionValue("i"); + String processor = cmd.getOptionValue("p", "Ls"); + String outputFile = cmd.getOptionValue("o"); + + PrintWriter out = (outputFile == null || outputFile.equals("-")) ? new PrintWriter( + System.out) : new PrintWriter(new File(outputFile)); + + Configuration conf = new Configuration(); + try { + if (processor.equals("FileDistribution")) { + long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0")); + int step = Integer.parseInt(cmd.getOptionValue("step", "0")); + new FileDistributionCalculator(conf, maxSize, step, out) + .visit(new RandomAccessFile(inputFile, "r")); + } else if (processor.equals("XML")) { + new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile, + "r")); + } else { + new LsrPBImage(conf, out).visit(new RandomAccessFile(inputFile, "r")); + } + } catch (EOFException e) { + System.err.println("Input file ended unexpectedly. Exiting"); + } catch (IOException e) { + System.err.println("Encountered exception. Exiting: " + e.getMessage()); + } finally { + out.close(); + } + + } + + /** + * Print application usage instructions. + */ + private static void printUsage() { + System.out.println(usage); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java new file mode 100644 index 00000000000..7ebf1196c4b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -0,0 +1,415 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; +import org.apache.hadoop.hdfs.server.namenode.FSImageUtil; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeSymlink; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; +import org.apache.hadoop.io.IOUtils; + +import com.google.common.collect.Lists; +import com.google.common.io.LimitInputStream; + +/** + * This is the tool for analyzing file sizes in the namespace image. In order to + * run the tool one should define a range of integers [0, maxSize] by + * specifying maxSize and a step. The range of integers is + * divided into segments of size step: + * [0, s1, ..., sn-1, maxSize], and the visitor + * calculates how many files in the system fall into each segment + * [si-1, si). Note that files larger than + * maxSize always fall into the very last segment. + * + *

Input.

+ *
    + *
  • filename specifies the location of the image file;
    • + *
  • maxSize determines the range [0, maxSize] of files + * sizes considered by the visitor;
    • + *
  • step the range is divided into segments of size step.
    • + *
+ * + *

Output.

The output file is formatted as a tab separated two column + * table: Size and NumFiles. Where Size represents the start of the segment, and + * numFiles is the number of files form the image which size falls in this + * segment. + * + */ +@InterfaceAudience.Private +public final class PBImageXmlWriter { + private final Configuration conf; + private final PrintWriter out; + private String[] stringTable; + + public PBImageXmlWriter(Configuration conf, PrintWriter out) { + this.conf = conf; + this.out = out; + } + + public void visit(RandomAccessFile file) throws IOException { + if (!FSImageUtil.checkFileFormat(file)) { + throw new IOException("Unrecognized FSImage"); + } + + FileSummary summary = FSImageUtil.loadSummary(file); + FileInputStream fin = null; + try { + fin = new FileInputStream(file.getFD()); + out.print("\n"); + + ArrayList sections = Lists.newArrayList(summary + .getSectionsList()); + Collections.sort(sections, new Comparator() { + @Override + public int compare(FileSummary.Section s1, FileSummary.Section s2) { + SectionName n1 = SectionName.fromString(s1.getName()); + SectionName n2 = SectionName.fromString(s2.getName()); + if (n1 == null) { + return n2 == null ? 0 : -1; + } else if (n2 == null) { + return -1; + } else { + return n1.ordinal() - n2.ordinal(); + } + } + }); + + for (FileSummary.Section s : sections) { + fin.getChannel().position(s.getOffset()); + InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, + summary.getCodec(), new BufferedInputStream(new LimitInputStream( + fin, s.getLength()))); + + switch (SectionName.fromString(s.getName())) { + case NS_INFO: + dumpNameSection(is); + break; + case STRING_TABLE: + loadStringTable(is); + break; + case INODE: + dumpINodeSection(is); + break; + case INODE_DIR: + dumpINodeDirectorySection(is); + break; + case FILES_UNDERCONSTRUCTION: + dumpFileUnderConstructionSection(is); + break; + case SNAPSHOT: + dumpSnapshotSection(is); + break; + case SNAPSHOT_DIFF: + dumpSnapshotDiffSection(is); + break; + case SECRET_MANAGER: + dumpSecretManagerSection(is); + break; + case CACHE_MANAGER: + dumpCacheManagerSection(is); + break; + default: + break; + } + } + } finally { + IOUtils.cleanup(null, fin); + } + } + + private void dumpCacheManagerSection(InputStream is) throws IOException { + out.print(""); + CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(is); + o("nextDirectiveId", s.getNextDirectiveId()); + for (int i = 0; i < s.getNumPools(); ++i) { + CachePoolInfoProto p = CachePoolInfoProto.parseDelimitedFrom(is); + out.print(""); + o("poolName", p.getPoolName()).o("ownerName", p.getOwnerName()) + .o("groupName", p.getGroupName()).o("mode", p.getMode()) + .o("limit", p.getLimit()) + .o("maxRelativeExpiry", p.getMaxRelativeExpiry()); + out.print("\n"); + } + for (int i = 0; i < s.getNumPools(); ++i) { + CacheDirectiveInfoProto p = CacheDirectiveInfoProto + .parseDelimitedFrom(is); + out.print(""); + o("id", p.getId()).o("path", p.getPath()) + .o("replication", p.getReplication()).o("pool", p.getPool()); + out.print(""); + CacheDirectiveInfoExpirationProto e = p.getExpiration(); + o("millis", e.getMillis()).o("relatilve", e.getIsRelative()); + out.print("\n"); + out.print("\n"); + } + out.print("\n"); + + } + + private void dumpFileUnderConstructionSection(InputStream in) + throws IOException { + out.print(""); + while (true) { + FileUnderConstructionEntry e = FileUnderConstructionEntry + .parseDelimitedFrom(in); + if (e == null) { + break; + } + out.print(""); + o("id", e.getInodeId()).o("path", e.getFullPath()); + out.print("\n"); + } + out.print("\n"); + } + + private void dumpINodeDirectory(INodeDirectory d) { + o("mtime", d.getModificationTime()).o("permission", + dumpPermission(d.getPermission())); + + if (d.hasDsQuota() && d.hasNsQuota()) { + o("nsquota", d.getNsQuota()).o("dsquota", d.getDsQuota()); + } + } + + private void dumpINodeDirectorySection(InputStream in) throws IOException { + out.print(""); + while (true) { + INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry + .parseDelimitedFrom(in); + // note that in is a LimitedInputStream + if (e == null) { + break; + } + out.print(""); + o("parent", e.getParent()); + for (long id : e.getChildrenList()) { + o("inode", id); + } + for (int i = 0; i < e.getNumOfRef(); i++) { + INodeSection.INodeReference r = INodeSection.INodeReference + .parseDelimitedFrom(in); + dumpINodeReference(r); + + } + out.print("\n"); + } + out.print("\n"); + } + + private void dumpINodeReference(INodeSection.INodeReference r) { + out.print(""); + o("referredId", r.getReferredId()).o("name", r.getName().toStringUtf8()) + .o("dstSnapshotId", r.getDstSnapshotId()) + .o("lastSnapshotId", r.getLastSnapshotId()); + out.print("\n"); + } + + private void dumpINodeFile(INodeSection.INodeFile f) { + o("replication", f.getReplication()).o("mtime", f.getModificationTime()) + .o("atime", f.getAccessTime()) + .o("perferredBlockSize", f.getPreferredBlockSize()) + .o("permission", dumpPermission(f.getPermission())); + + if (f.getBlocksCount() > 0) { + out.print(""); + for (BlockProto b : f.getBlocksList()) { + out.print(""); + o("id", b.getBlockId()).o("genstamp", b.getGenStamp()).o("numBytes", + b.getNumBytes()); + out.print("\n"); + } + out.print("\n"); + } + + if (f.hasFileUC()) { + INodeSection.FileUnderConstructionFeature u = f.getFileUC(); + out.print(""); + o("clientName", u.getClientName()).o("clientMachine", + u.getClientMachine()); + out.print("\n"); + } + } + + private void dumpINodeSection(InputStream in) throws IOException { + INodeSection s = INodeSection.parseDelimitedFrom(in); + out.print(""); + o("lastInodeId", s.getLastInodeId()); + for (int i = 0; i < s.getNumInodes(); ++i) { + INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); + out.print(""); + o("id", p.getId()).o("type", p.getType()).o("name", + p.getName().toStringUtf8()); + + if (p.hasFile()) { + dumpINodeFile(p.getFile()); + } else if (p.hasDirectory()) { + dumpINodeDirectory(p.getDirectory()); + } else if (p.hasSymlink()) { + dumpINodeSymlink(p.getSymlink()); + } + + out.print("\n"); + } + out.print("\n"); + } + + private void dumpINodeSymlink(INodeSymlink s) { + o("permission", dumpPermission(s.getPermission())).o("target", + s.getTarget().toStringUtf8()); + } + + private void dumpNameSection(InputStream in) throws IOException { + NameSystemSection s = NameSystemSection.parseDelimitedFrom(in); + out.print("\n"); + o("genstampV1", s.getGenstampV1()).o("genstampV2", s.getGenstampV2()) + .o("genstampV1Limit", s.getGenstampV1Limit()) + .o("lastAllocatedBlockId", s.getLastAllocatedBlockId()) + .o("txid", s.getTransactionId()); + out.print("\n"); + } + + private String dumpPermission(long permission) { + return FSImageFormatPBINode.Loader.loadPermission(permission, stringTable) + .toString(); + } + + private void dumpSecretManagerSection(InputStream is) throws IOException { + out.print(""); + SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(is); + o("currentId", s.getCurrentId()).o("tokenSequenceNumber", + s.getTokenSequenceNumber()); + out.print(""); + } + + private void dumpSnapshotDiffSection(InputStream in) throws IOException { + out.print(""); + while (true) { + SnapshotDiffSection.DiffEntry e = SnapshotDiffSection.DiffEntry + .parseDelimitedFrom(in); + if (e == null) { + break; + } + out.print(""); + o("inodeid", e.getInodeId()); + switch (e.getType()) { + case FILEDIFF: { + for (int i = 0; i < e.getNumOfDiff(); ++i) { + out.print(""); + SnapshotDiffSection.FileDiff f = SnapshotDiffSection.FileDiff + .parseDelimitedFrom(in); + o("snapshotId", f.getSnapshotId()).o("size", f.getFileSize()).o( + "name", f.getName().toStringUtf8()); + out.print("\n"); + } + } + break; + case DIRECTORYDIFF: { + for (int i = 0; i < e.getNumOfDiff(); ++i) { + out.print(""); + SnapshotDiffSection.DirectoryDiff d = SnapshotDiffSection.DirectoryDiff + .parseDelimitedFrom(in); + o("snapshotId", d.getSnapshotId()) + .o("isSnapshotroot", d.getIsSnapshotRoot()) + .o("childrenSize", d.getChildrenSize()) + .o("name", d.getName().toStringUtf8()); + + for (int j = 0; j < d.getCreatedListSize(); ++j) { + SnapshotDiffSection.CreatedListEntry ce = SnapshotDiffSection.CreatedListEntry + .parseDelimitedFrom(in); + out.print(""); + o("name", ce.getName().toStringUtf8()); + out.print("\n"); + } + for (int j = 0; j < d.getNumOfDeletedRef(); ++j) { + INodeSection.INodeReference r = INodeSection.INodeReference + .parseDelimitedFrom(in); + dumpINodeReference(r); + } + out.print("\n"); + } + } + break; + default: + break; + } + out.print(""); + } + out.print("\n"); + } + + private void dumpSnapshotSection(InputStream in) throws IOException { + out.print(""); + SnapshotSection s = SnapshotSection.parseDelimitedFrom(in); + o("snapshotCounter", s.getSnapshotCounter()); + if (s.getSnapshottableDirCount() > 0) { + out.print(""); + for (long id : s.getSnapshottableDirList()) { + o("dir", id); + } + out.print("\n"); + } + for (int i = 0; i < s.getNumSnapshots(); ++i) { + SnapshotSection.Snapshot pbs = SnapshotSection.Snapshot + .parseDelimitedFrom(in); + o("snapshot", pbs.getSnapshotId()); + } + out.print("\n"); + } + + private void loadStringTable(InputStream in) throws IOException { + StringTableSection s = StringTableSection.parseDelimitedFrom(in); + stringTable = new String[s.getNumEntry() + 1]; + for (int i = 0; i < s.getNumEntry(); ++i) { + StringTableSection.Entry e = StringTableSection.Entry + .parseDelimitedFrom(in); + stringTable[e.getId()] = e.getStr(); + } + } + + private PBImageXmlWriter o(final String e, final Object v) { + out.print("<" + e + ">" + v + ""); + return this; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto new file mode 100644 index 00000000000..af7ba874d29 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto @@ -0,0 +1,280 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.hdfs.server.namenode"; +option java_outer_classname = "FsImageProto"; + +package hadoop.hdfs.fsimage; + +import "hdfs.proto"; + +/** + * This file defines the on-disk layout of the file system image. The + * layout is defined by the following EBNF grammar, in which angle + * brackets mark protobuf definitions. (e.g., ) + * + * FILE := MAGIC SECTION* FileSummaryLength + * MAGIC := 'HDFSIMG1' + * SECTION := | ... + * FileSummaryLength := 4 byte int + * + * Some notes: + * + * The codec field in FileSummary describes the compression codec used + * for all sections. The fileheader is always uncompressed. + * + * All protobuf messages are serialized in delimited form, which means + * that there always will be an integer indicates the size of the + * protobuf message. + * + */ + +message FileSummary { + // The version of the above EBNF grammars. + required uint32 ondiskVersion = 1; + // layoutVersion describes which features are available in the + // FSImage. + required uint32 layoutVersion = 2; + optional string codec = 3; + // index for each section + message Section { + optional string name = 1; + optional uint64 length = 2; + optional uint64 offset = 3; + } + repeated Section sections = 4; +} + +/** + * Name: NS_INFO + */ +message NameSystemSection { + optional uint32 namespaceId = 1; + optional uint64 genstampV1 = 2; + optional uint64 genstampV2 = 3; + optional uint64 genstampV1Limit = 4; + optional uint64 lastAllocatedBlockId = 5; + optional uint64 transactionId = 6; +} + +/** + * Permission is serialized as a 64-bit long. [0:24):[25:48):[48:64) (in Big Endian). + * The first and the second parts are the string ids of the user and + * group name, and the last 16 bits are the permission bits. + * + * Name: INODE + */ +message INodeSection { + /** + * under-construction feature for INodeFile + */ + message FileUnderConstructionFeature { + optional string clientName = 1; + optional string clientMachine = 2; + } + + message INodeFile { + optional uint32 replication = 1; + optional uint64 modificationTime = 2; + optional uint64 accessTime = 3; + optional uint64 preferredBlockSize = 4; + optional fixed64 permission = 5; + repeated BlockProto blocks = 6; + optional FileUnderConstructionFeature fileUC = 7; + } + + message INodeDirectory { + optional uint64 modificationTime = 1; + // namespace quota + optional uint64 nsQuota = 2; + // diskspace quota + optional uint64 dsQuota = 3; + optional fixed64 permission = 4; + } + + message INodeSymlink { + optional fixed64 permission = 1; + optional bytes target = 2; + } + + message INodeReference { + // id of the referred inode + optional uint64 referredId = 1; + // local name recorded in WithName + optional bytes name = 2; + // recorded in DstReference + optional uint32 dstSnapshotId = 3; + // recorded in WithName + optional uint32 lastSnapshotId = 4; + } + + message INode { + enum Type { + FILE = 1; + DIRECTORY = 2; + SYMLINK = 3; + }; + required Type type = 1; + required uint64 id = 2; + optional bytes name = 3; + + optional INodeFile file = 4; + optional INodeDirectory directory = 5; + optional INodeSymlink symlink = 6; + } + + optional uint64 lastInodeId = 1; + optional uint64 numInodes = 2; + // repeated INodes.. +} + +/** + * This section records information about under-construction files for + * reconstructing the lease map. + * NAME: FILES_UNDERCONSTRUCTION + */ +message FilesUnderConstructionSection { + message FileUnderConstructionEntry { + optional uint64 inodeId = 1; + optional string fullPath = 2; + } + // repeated FileUnderConstructionEntry... +} + +/** + * This section records the children of each directories + * NAME: INODE_DIR + */ +message INodeDirectorySection { + message DirEntry { + optional uint64 parent = 1; + repeated uint64 children = 2 [packed = true]; + optional uint64 numOfRef = 3; + // repeated INodeReference... + } + // repeated DirEntry, ended at the boundary of the section. +} + +/** + * This section records the information about snapshot + * NAME: SNAPSHOT + */ +message SnapshotSection { + message Snapshot { + optional uint32 snapshotId = 1; + // Snapshot root + optional INodeSection.INode root = 2; + } + + optional uint32 snapshotCounter = 1; + repeated uint64 snapshottableDir = 2 [packed = true]; + // total number of snapshots + optional uint32 numSnapshots = 3; + // repeated Snapshot... +} + +/** + * This section records information about snapshot diffs + * NAME: SNAPSHOT_DIFF + */ +message SnapshotDiffSection { + message CreatedListEntry { + optional bytes name = 1; + } + + message DirectoryDiff { + optional uint32 snapshotId = 1; + optional uint32 childrenSize = 2; + optional bool isSnapshotRoot = 3; + optional bytes name = 4; + optional INodeSection.INodeDirectory snapshotCopy = 5; + optional uint32 createdListSize = 6; + optional uint32 numOfDeletedRef = 7; // number of reference nodes in deleted list + repeated uint64 deletedINode = 8 [packed = true]; // id of deleted inode + // repeated CreatedListEntry (size is specified by createdListSize) + // repeated INodeReference (reference inodes in deleted list) + } + + message FileDiff { + optional uint32 snapshotId = 1; + optional uint64 fileSize = 2; + optional bytes name = 3; + optional INodeSection.INodeFile snapshotCopy = 4; + } + + message DiffEntry { + enum Type { + FILEDIFF = 1; + DIRECTORYDIFF = 2; + } + required Type type = 1; + optional uint64 inodeId = 2; + optional uint32 numOfDiff = 3; + + // repeated DirectoryDiff or FileDiff + } + + // repeated DiffEntry +} + +/** + * This section maps string to id + * NAME: STRING_TABLE + */ +message StringTableSection { + message Entry { + optional uint32 id = 1; + optional string str = 2; + } + optional uint32 numEntry = 1; + // repeated Entry +} + +message SecretManagerSection { + message DelegationKey { + optional uint32 id = 1; + optional uint64 expiryDate = 2; + optional bytes key = 3; + } + message PersistToken { + optional uint32 version = 1; + optional string owner = 2; + optional string renewer = 3; + optional string realUser = 4; + optional uint64 issueDate = 5; + optional uint64 maxDate = 6; + optional uint32 sequenceNumber = 7; + optional uint32 masterKeyId = 8; + optional uint64 expiryDate = 9; + } + optional uint32 currentId = 1; + optional uint32 tokenSequenceNumber = 2; + optional uint32 numKeys = 3; + optional uint32 numTokens = 4; + // repeated DelegationKey keys + // repeated PersistToken tokens +} + +message CacheManagerSection { + required uint64 nextDirectiveId = 1; + required uint32 numPools = 2; + required uint32 numDirectives = 3; + // repeated CachePoolInfoProto pools + // repeated CacheDirectiveInfoProto directives +} + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java new file mode 100644 index 00000000000..552b091b7b4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.EnumSet; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; +import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; +import org.apache.hadoop.hdfs.util.MD5FileUtils; +import org.junit.Test; + +public class TestFSImage { + + @Test + public void testPersist() throws IOException { + Configuration conf = new Configuration(); + testPersistHelper(conf); + } + + @Test + public void testCompression() throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true); + conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, + "org.apache.hadoop.io.compress.GzipCodec"); + testPersistHelper(conf); + } + + private void testPersistHelper(Configuration conf) throws IOException { + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + FSNamesystem fsn = cluster.getNamesystem(); + DistributedFileSystem fs = cluster.getFileSystem(); + + final Path dir = new Path("/abc/def"); + final Path file1 = new Path(dir, "f1"); + final Path file2 = new Path(dir, "f2"); + + // create an empty file f1 + fs.create(file1).close(); + + // create an under-construction file f2 + FSDataOutputStream out = fs.create(file2); + out.writeBytes("hello"); + ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet + .of(SyncFlag.UPDATE_LENGTH)); + + // checkpoint + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.restartNameNode(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + + assertTrue(fs.isDirectory(dir)); + assertTrue(fs.exists(file1)); + assertTrue(fs.exists(file2)); + + // check internals of file2 + INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile(); + assertEquals("hello".length(), file2Node.computeFileSize()); + assertTrue(file2Node.isUnderConstruction()); + BlockInfo[] blks = file2Node.getBlocks(); + assertEquals(1, blks.length); + assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState()); + // check lease manager + Lease lease = fsn.leaseManager.getLeaseByPath(file2.toString()); + Assert.assertNotNull(lease); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + /** + * Ensure that the digest written by the saver equals to the digest of the + * file. + */ + @Test + public void testDigest() throws IOException { + Configuration conf = new Configuration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + DistributedFileSystem fs = cluster.getFileSystem(); + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get( + 0); + File fsimage = FSImageTestUtil.findNewestImageFile(currentDir + .getAbsolutePath()); + assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage), + MD5FileUtils.computeMd5ForFile(fsimage)); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java index 21935d05d9c..f3cbf15aae2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java @@ -140,7 +140,7 @@ public class TestFSImageWithSnapshot { private File saveFSImageToTempFile() throws IOException { SaveNamespaceContext context = new SaveNamespaceContext(fsn, txid, new Canceler()); - FSImageFormat.Saver saver = new FSImageFormat.Saver(context); + FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); File imageFile = getImageFile(testDir, txid); fsn.readLock(); @@ -154,7 +154,7 @@ public class TestFSImageWithSnapshot { /** Load the fsimage from a temp file */ private void loadFSImageFromTempFile(File imageFile) throws IOException { - FSImageFormat.Loader loader = new FSImageFormat.Loader(conf, fsn); + FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, fsn); fsn.writeLock(); fsn.getFSDirectory().writeLock(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java index 3ff5d54dc66..0ca112da5c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java @@ -287,7 +287,6 @@ public class TestStandbyCheckpoints { doEdits(0, 1000); nn0.getRpcServer().rollEditLog(); answerer.waitForCall(); - answerer.proceed(); assertTrue("SBN is not performing checkpoint but it should be.", answerer.getFireCount() == 1 && answerer.getResultCount() == 0); @@ -306,6 +305,7 @@ public class TestStandbyCheckpoints { // RPC to the SBN happened during the checkpoint. assertTrue("SBN should have still been checkpointing.", answerer.getFireCount() == 1 && answerer.getResultCount() == 0); + answerer.proceed(); answerer.waitForResult(); assertTrue("SBN should have finished checkpointing.", answerer.getFireCount() == 1 && answerer.getResultCount() == 1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index 7fe8087f2a4..d4e887949e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -73,7 +73,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -; /** Testing rename with snapshots. */ public class TestRenameWithSnapshots { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java index a6517258367..37f94f5fd12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java @@ -25,6 +25,9 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -53,8 +56,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree.Node; -import org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer; -import org.apache.hadoop.hdfs.tools.offlineImageViewer.XmlImageVisitor; +import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -245,8 +247,8 @@ public class TestSnapshot { * snapshots */ @Test - public void testOfflineImageViewer() throws Throwable { - runTestSnapshot(SNAPSHOT_ITERATION_NUMBER); + public void testOfflineImageViewer() throws Exception { + runTestSnapshot(1); // retrieve the fsimage. Note that we already save namespace to fsimage at // the end of each iteration of runTestSnapshot. @@ -254,31 +256,10 @@ public class TestSnapshot { FSImageTestUtil.getFSImage( cluster.getNameNode()).getStorage().getStorageDir(0)); assertNotNull("Didn't generate or can't find fsimage", originalFsimage); - - String ROOT = System.getProperty("test.build.data", "build/test/data"); - File testFile = new File(ROOT, "/image"); - String xmlImage = ROOT + "/image_xml"; - boolean success = false; - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - XmlImageVisitor v = new XmlImageVisitor(xmlImage, true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, - true); - oiv.go(); - success = true; - } finally { - if (testFile.exists()) { - testFile.delete(); - } - // delete the xml file if the parsing is successful - if (success) { - File xmlImageFile = new File(xmlImage); - if (xmlImageFile.exists()) { - xmlImageFile.delete(); - } - } - } + StringWriter output = new StringWriter(); + PrintWriter o = new PrintWriter(output); + PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o); + v.visit(new RandomAccessFile(originalFsimage, "r")); } private void runTestSnapshot(int iteration) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 11aa3b821f0..91a5c1521c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -20,23 +20,20 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.io.StringWriter; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -46,27 +43,29 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.test.PathUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; - +import org.junit.rules.TemporaryFolder; /** - * Test function of OfflineImageViewer by: - * * confirming it can correctly process a valid fsimage file and that - * the processing generates a correct representation of the namespace - * * confirming it correctly fails to process an fsimage file with a layout - * version it shouldn't be able to handle - * * confirm it correctly bails on malformed image files, in particular, a - * file that ends suddenly. + * Test function of OfflineImageViewer by: * confirming it can correctly process + * a valid fsimage file and that the processing generates a correct + * representation of the namespace * confirming it correctly fails to process an + * fsimage file with a layout version it shouldn't be able to handle * confirm + * it correctly bails on malformed image files, in particular, a file that ends + * suddenly. */ public class TestOfflineImageViewer { private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class); @@ -76,22 +75,22 @@ public class TestOfflineImageViewer { private static File originalFsimage = null; // Elements of lines of ls-file output to be compared to FileStatus instance - private static class LsElements { - public String perms; - public int replication; - public String username; - public String groupname; - public long filesize; - public char dir; // d if dir, - otherwise + private static final class LsElements { + private String perms; + private int replication; + private String username; + private String groupname; + private long filesize; + private boolean isDir; } - + // namespace as written to dfs, to be compared with viewer's output - final static HashMap writtenFiles = - new HashMap(); - - private static String ROOT = PathUtils.getTestDirName(TestOfflineImageViewer.class); - - // Create a populated namespace for later testing. Save its contents to a + final static HashMap writtenFiles = new HashMap(); + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + // Create a populated namespace for later testing. Save its contents to a // data structure and store its fsimage location. // We only want to generate the fsimage file once and use it for // multiple tests. @@ -100,35 +99,39 @@ public class TestOfflineImageViewer { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); cluster.waitActive(); FileSystem hdfs = cluster.getFileSystem(); - + int filesize = 256; - - // Create a reasonable namespace - for(int i = 0; i < NUM_DIRS; i++) { + + // Create a reasonable namespace + for (int i = 0; i < NUM_DIRS; i++) { Path dir = new Path("/dir" + i); hdfs.mkdirs(dir); writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString())); - for(int j = 0; j < FILES_PER_DIR; j++) { + for (int j = 0; j < FILES_PER_DIR; j++) { Path file = new Path(dir, "file" + j); FSDataOutputStream o = hdfs.create(file); - o.write(new byte[ filesize++ ]); + o.write(new byte[filesize++]); o.close(); - - writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString())); + + writtenFiles.put(file.toString(), + pathToFileEntry(hdfs, file.toString())); } } // Get delegation tokens so we log the delegation token op - Token[] delegationTokens = - hdfs.addDelegationTokens(TEST_RENEWER, null); + Token[] delegationTokens = hdfs + .addDelegationTokens(TEST_RENEWER, null); for (Token t : delegationTokens) { LOG.debug("got token " + t); } @@ -137,329 +140,113 @@ public class TestOfflineImageViewer { cluster.getNameNodeRpc() .setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); cluster.getNameNodeRpc().saveNamespace(); - + // Determine location of fsimage file - originalFsimage = FSImageTestUtil.findLatestImageFile( - FSImageTestUtil.getFSImage( - cluster.getNameNode()).getStorage().getStorageDir(0)); + originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); if (originalFsimage == null) { throw new RuntimeException("Didn't generate or can't find fsimage"); } LOG.debug("original FS image file is " + originalFsimage); } finally { - if(cluster != null) + if (cluster != null) cluster.shutdown(); } } - + @AfterClass public static void deleteOriginalFSImage() throws IOException { - if(originalFsimage != null && originalFsimage.exists()) { + if (originalFsimage != null && originalFsimage.exists()) { originalFsimage.delete(); } } - - // Convenience method to generate a file status from file system for + + // Convenience method to generate a file status from file system for // later comparison - private static FileStatus pathToFileEntry(FileSystem hdfs, String file) - throws IOException { + private static FileStatus pathToFileEntry(FileSystem hdfs, String file) + throws IOException { return hdfs.getFileStatus(new Path(file)); } - - // Verify that we can correctly generate an ls-style output for a valid + + // Verify that we can correctly generate an ls-style output for a valid // fsimage @Test public void outputOfLSVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - File outputFile = new File(ROOT, "/basicCheckOutput"); - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - - oiv.go(); - - HashMap fileOutput = readLsfile(outputFile); - - compareNamespaces(writtenFiles, fileOutput); - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } - LOG.debug("Correctly generated ls-style output."); - } - - // Confirm that attempting to read an fsimage file with an unsupported - // layout results in an error - @Test - public void unsupportedFSLayoutVersion() throws IOException { - File testFile = new File(ROOT, "/invalidLayoutVersion"); - File outputFile = new File(ROOT, "invalidLayoutVersionOutput"); - - try { - int badVersionNum = -432; - changeLayoutVersion(originalFsimage, testFile, badVersionNum); - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - - try { - oiv.go(); - fail("Shouldn't be able to read invalid laytout version"); - } catch(IOException e) { - if(!e.getMessage().contains(Integer.toString(badVersionNum))) - throw e; // wasn't error we were expecting - LOG.debug("Correctly failed at reading bad image version."); + StringWriter output = new StringWriter(); + PrintWriter out = new PrintWriter(output); + LsrPBImage v = new LsrPBImage(new Configuration(), out); + v.visit(new RandomAccessFile(originalFsimage, "r")); + out.close(); + Pattern pattern = Pattern + .compile("([d\\-])([rwx\\-]{9})\\s*(-|\\d+)\\s*(\\w+)\\s*(\\w+)\\s*(\\d+)\\s*(\\d+)\\s*([\b/]+)"); + int count = 0; + for (String s : output.toString().split("\n")) { + Matcher m = pattern.matcher(s); + assertTrue(m.find()); + LsElements e = new LsElements(); + e.isDir = m.group(1).equals("d"); + e.perms = m.group(2); + e.replication = m.group(3).equals("-") ? 0 : Integer.parseInt(m.group(3)); + e.username = m.group(4); + e.groupname = m.group(5); + e.filesize = Long.parseLong(m.group(7)); + String path = m.group(8); + if (!path.equals("/")) { + compareFiles(writtenFiles.get(path), e); } - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); + ++count; } + assertEquals(writtenFiles.size() + 1, count); } - - // Verify that image viewer will bail on a file that ends unexpectedly - @Test - public void truncatedFSImage() throws IOException { - File testFile = new File(ROOT, "/truncatedFSImage"); - File outputFile = new File(ROOT, "/trucnatedFSImageOutput"); - try { - copyPartOfFile(originalFsimage, testFile); - assertTrue("Created truncated fsimage", testFile.exists()); - - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - try { - oiv.go(); - fail("Managed to process a truncated fsimage file"); - } catch (EOFException e) { - LOG.debug("Correctly handled EOF"); - } - - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } + @Test(expected = IOException.class) + public void testTruncatedFSImage() throws IOException { + File truncatedFile = folder.newFile(); + StringWriter output = new StringWriter(); + copyPartOfFile(originalFsimage, truncatedFile); + new FileDistributionCalculator(new Configuration(), 0, 0, new PrintWriter( + output)).visit(new RandomAccessFile(truncatedFile, "r")); } - - // Test that our ls file has all the same compenents of the original namespace - private void compareNamespaces(HashMap written, - HashMap fileOutput) { - assertEquals( "Should be the same number of files in both, plus one for root" - + " in fileoutput", fileOutput.keySet().size(), - written.keySet().size() + 1); - Set inFile = fileOutput.keySet(); - // For each line in the output file, verify that the namespace had a - // filestatus counterpart - for (String path : inFile) { - if (path.equals("/")) // root's not included in output from system call - continue; - - assertTrue("Path in file (" + path + ") was written to fs", written - .containsKey(path)); - - compareFiles(written.get(path), fileOutput.get(path)); - - written.remove(path); - } - - assertEquals("No more files were written to fs", 0, written.size()); - } - // Compare two files as listed in the original namespace FileStatus and // the output of the ls file from the image processor private void compareFiles(FileStatus fs, LsElements elements) { - assertEquals("directory listed as such", - fs.isDirectory() ? 'd' : '-', elements.dir); - assertEquals("perms string equal", - fs.getPermission().toString(), elements.perms); + assertEquals("directory listed as such", fs.isDirectory(), elements.isDir); + assertEquals("perms string equal", fs.getPermission().toString(), + elements.perms); assertEquals("replication equal", fs.getReplication(), elements.replication); assertEquals("owner equal", fs.getOwner(), elements.username); assertEquals("group equal", fs.getGroup(), elements.groupname); assertEquals("lengths equal", fs.getLen(), elements.filesize); } - // Read the contents of the file created by the Ls processor - private HashMap readLsfile(File lsFile) throws IOException { - BufferedReader br = new BufferedReader(new FileReader(lsFile)); - String line = null; - HashMap fileContents = new HashMap(); - - while((line = br.readLine()) != null) - readLsLine(line, fileContents); - - br.close(); - return fileContents; - } - - // Parse a line from the ls output. Store permissions, replication, - // username, groupname and filesize in hashmap keyed to the path name - private void readLsLine(String line, HashMap fileContents) { - String elements [] = line.split("\\s+"); - - assertEquals("Not enough elements in ls output", 8, elements.length); - - LsElements lsLine = new LsElements(); - - lsLine.dir = elements[0].charAt(0); - lsLine.perms = elements[0].substring(1); - lsLine.replication = elements[1].equals("-") - ? 0 : Integer.valueOf(elements[1]); - lsLine.username = elements[2]; - lsLine.groupname = elements[3]; - lsLine.filesize = Long.valueOf(elements[4]); - // skipping date and time - - String path = elements[7]; - - // Check that each file in the ls output was listed once - assertFalse("LS file had duplicate file entries", - fileContents.containsKey(path)); - - fileContents.put(path, lsLine); - } - - // Copy one fsimage to another, changing the layout version in the process - private void changeLayoutVersion(File src, File dest, int newVersion) - throws IOException { - DataInputStream in = null; - DataOutputStream out = null; - - try { - in = new DataInputStream(new FileInputStream(src)); - out = new DataOutputStream(new FileOutputStream(dest)); - - in.readInt(); - out.writeInt(newVersion); - - byte [] b = new byte[1024]; - while( in.read(b) > 0 ) { - out.write(b); - } - } finally { - if(in != null) in.close(); - if(out != null) out.close(); - } - } - - // Only copy part of file into the other. Used for testing truncated fsimage private void copyPartOfFile(File src, File dest) throws IOException { - InputStream in = null; - OutputStream out = null; - - byte [] b = new byte[256]; - int bytesWritten = 0; - int count; - int maxBytes = 700; - + FileInputStream in = null; + FileOutputStream out = null; + final int MAX_BYTES = 700; try { in = new FileInputStream(src); out = new FileOutputStream(dest); - - while( (count = in.read(b)) > 0 && bytesWritten < maxBytes ) { - out.write(b); - bytesWritten += count; - } + in.getChannel().transferTo(0, MAX_BYTES, out.getChannel()); } finally { - if(in != null) in.close(); - if(out != null) out.close(); + IOUtils.cleanup(null, in); + IOUtils.cleanup(null, out); } } @Test - public void outputOfFileDistributionVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - File outputFile = new File(ROOT, "/fileDistributionCheckOutput"); + public void testFileDistributionVisitor() throws IOException { + StringWriter output = new StringWriter(); + PrintWriter o = new PrintWriter(output); + new FileDistributionCalculator(new Configuration(), 0, 0, o) + .visit(new RandomAccessFile(originalFsimage, "r")); + o.close(); - int totalFiles = 0; - BufferedReader reader = null; - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - ImageVisitor v = new FileDistributionVisitor(outputFile.getPath(), 0, 0); - OfflineImageViewer oiv = - new OfflineImageViewer(testFile.getPath(), v, false); + Pattern p = Pattern.compile("totalFiles = (\\d+)\n"); + Matcher matcher = p.matcher(output.getBuffer()); - oiv.go(); - - reader = new BufferedReader(new FileReader(outputFile)); - String line = reader.readLine(); - assertEquals(line, "Size\tNumFiles"); - while((line = reader.readLine()) != null) { - String[] row = line.split("\t"); - assertEquals(row.length, 2); - totalFiles += Integer.parseInt(row[1]); - } - } finally { - if (reader != null) { - reader.close(); - } - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } + assertTrue(matcher.find() && matcher.groupCount() == 1); + int totalFiles = Integer.parseInt(matcher.group(1)); assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR); } - - private static class TestImageVisitor extends ImageVisitor { - private List delegationTokenRenewers = new LinkedList(); - TestImageVisitor() { - } - - List getDelegationTokenRenewers() { - return delegationTokenRenewers; - } - - @Override - void start() throws IOException { - } - - @Override - void finish() throws IOException { - } - - @Override - void finishAbnormally() throws IOException { - } - - @Override - void visit(ImageElement element, String value) throws IOException { - if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) { - delegationTokenRenewers.add(value); - } - } - - @Override - void visitEnclosingElement(ImageElement element) throws IOException { - } - - @Override - void visitEnclosingElement(ImageElement element, ImageElement key, - String value) throws IOException { - } - - @Override - void leaveEnclosingElement() throws IOException { - } - } - - @Test - public void outputOfTestVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - TestImageVisitor v = new TestImageVisitor(); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true); - oiv.go(); - - // Validated stored delegation token identifiers. - List dtrs = v.getDelegationTokenRenewers(); - assertEquals(1, dtrs.size()); - assertEquals(TEST_RENEWER, dtrs.get(0)); - } finally { - if(testFile.exists()) testFile.delete(); - } - LOG.debug("Passed TestVisitor validation."); - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index 7c1782fe9e0..f042e8446dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -1,6 +1,6 @@ - -51 + -52 OP_START_LOG_SEGMENT