Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-8470. fsimage loading progress should update inode, delegation token and cache pool count. (Contributed by surendra singh lilhore)

Browse Source
This commit is contained in:
Vinayakumar B 2015-06-03 14:24:55 +05:30
parent 54f83d9bd9
commit e965dcec37
3 changed files with 36 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -840,6 +840,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8256. "-storagepolicies , -blockId ,-replicaDetails " options are missed HDFS-8256. "-storagepolicies , -blockId ,-replicaDetails " options are missed
out in usage and from documentation (J.Andreina via vinayakumarb) out in usage and from documentation (J.Andreina via vinayakumarb)
HDFS-8470. fsimage loading progress should update inode, delegation token and
cache pool count. (surendra singh lilhore via vinayakumarb)
Release 2.7.1 - UNRELEASED Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
View File

@ -58,6 +58,10 @@
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeEntryProto; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeEntryProto;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.hdfs.util.ReadOnlyList;
@ -251,11 +255,15 @@ void loadINodeDirectorySection(InputStream in) throws IOException {
} }
} }
void loadINodeSection(InputStream in) throws IOException { void loadINodeSection(InputStream in, StartupProgress prog,
Step currentStep) throws IOException {
INodeSection s = INodeSection.parseDelimitedFrom(in); INodeSection s = INodeSection.parseDelimitedFrom(in);
fsn.dir.resetLastInodeId(s.getLastInodeId()); fsn.dir.resetLastInodeId(s.getLastInodeId());
LOG.info("Loading " + s.getNumInodes() + " INodes."); long numInodes = s.getNumInodes();
for (int i = 0; i < s.getNumInodes(); ++i) { LOG.info("Loading " + numInodes + " INodes.");
prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, numInodes);
Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep);
for (int i = 0; i < numInodes; ++i) {
INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in);
if (p.getId() == INodeId.ROOT_INODE_ID) { if (p.getId() == INodeId.ROOT_INODE_ID) {
loadRootINode(p); loadRootINode(p);
@ -263,6 +271,7 @@ void loadINodeSection(InputStream in) throws IOException {
INode n = loadINode(p); INode n = loadINode(p);
dir.addToInodeMap(n); dir.addToInodeMap(n);
} }
counter.increment();
} }
} }

30
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
View File

@ -55,6 +55,7 @@
import org.apache.hadoop.hdfs.server.namenode.snapshot.FSImageFormatPBSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.FSImageFormatPBSnapshot;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.hdfs.util.MD5FileUtils;
@ -250,7 +251,7 @@ public int compare(FileSummary.Section s1, FileSummary.Section s2) {
case INODE: { case INODE: {
currentStep = new Step(StepType.INODES); currentStep = new Step(StepType.INODES);
prog.beginStep(Phase.LOADING_FSIMAGE, currentStep); prog.beginStep(Phase.LOADING_FSIMAGE, currentStep);
inodeLoader.loadINodeSection(in); inodeLoader.loadINodeSection(in, prog, currentStep);
} }
break; break;
case INODE_REFERENCE: case INODE_REFERENCE:
@ -272,14 +273,14 @@ public int compare(FileSummary.Section s1, FileSummary.Section s2) {
prog.endStep(Phase.LOADING_FSIMAGE, currentStep); prog.endStep(Phase.LOADING_FSIMAGE, currentStep);
Step step = new Step(StepType.DELEGATION_TOKENS); Step step = new Step(StepType.DELEGATION_TOKENS);
prog.beginStep(Phase.LOADING_FSIMAGE, step); prog.beginStep(Phase.LOADING_FSIMAGE, step);
loadSecretManagerSection(in); loadSecretManagerSection(in, prog, step);
prog.endStep(Phase.LOADING_FSIMAGE, step); prog.endStep(Phase.LOADING_FSIMAGE, step);
} }
break; break;
case CACHE_MANAGER: { case CACHE_MANAGER: {
Step step = new Step(StepType.CACHE_POOLS); Step step = new Step(StepType.CACHE_POOLS);
prog.beginStep(Phase.LOADING_FSIMAGE, step); prog.beginStep(Phase.LOADING_FSIMAGE, step);
loadCacheManagerSection(in); loadCacheManagerSection(in, prog, step);
prog.endStep(Phase.LOADING_FSIMAGE, step); prog.endStep(Phase.LOADING_FSIMAGE, step);
} }
break; break;
@ -316,7 +317,8 @@ private void loadStringTableSection(InputStream in) throws IOException {
} }
} }
private void loadSecretManagerSection(InputStream in) throws IOException { private void loadSecretManagerSection(InputStream in, StartupProgress prog,
Step currentStep) throws IOException {
SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(in); SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(in);
int numKeys = s.getNumKeys(), numTokens = s.getNumTokens(); int numKeys = s.getNumKeys(), numTokens = s.getNumTokens();
ArrayList<SecretManagerSection.DelegationKey> keys = Lists ArrayList<SecretManagerSection.DelegationKey> keys = Lists
@ -327,20 +329,30 @@ private void loadSecretManagerSection(InputStream in) throws IOException {
for (int i = 0; i < numKeys; ++i) for (int i = 0; i < numKeys; ++i)
keys.add(SecretManagerSection.DelegationKey.parseDelimitedFrom(in)); keys.add(SecretManagerSection.DelegationKey.parseDelimitedFrom(in));
for (int i = 0; i < numTokens; ++i) prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, numTokens);
Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep);
for (int i = 0; i < numTokens; ++i) {
tokens.add(SecretManagerSection.PersistToken.parseDelimitedFrom(in)); tokens.add(SecretManagerSection.PersistToken.parseDelimitedFrom(in));
counter.increment();
}
fsn.loadSecretManagerState(s, keys, tokens); fsn.loadSecretManagerState(s, keys, tokens);
} }
private void loadCacheManagerSection(InputStream in) throws IOException { private void loadCacheManagerSection(InputStream in, StartupProgress prog,
Step currentStep) throws IOException {
CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(in); CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(in);
ArrayList<CachePoolInfoProto> pools = Lists.newArrayListWithCapacity(s int numPools = s.getNumPools();
.getNumPools()); ArrayList<CachePoolInfoProto> pools = Lists
.newArrayListWithCapacity(numPools);
ArrayList<CacheDirectiveInfoProto> directives = Lists ArrayList<CacheDirectiveInfoProto> directives = Lists
.newArrayListWithCapacity(s.getNumDirectives()); .newArrayListWithCapacity(s.getNumDirectives());
for (int i = 0; i < s.getNumPools(); ++i) prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, numPools);
Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep);
for (int i = 0; i < numPools; ++i) {
pools.add(CachePoolInfoProto.parseDelimitedFrom(in)); pools.add(CachePoolInfoProto.parseDelimitedFrom(in));
counter.increment();
}
for (int i = 0; i < s.getNumDirectives(); ++i) for (int i = 0; i < s.getNumDirectives(); ++i)
directives.add(CacheDirectiveInfoProto.parseDelimitedFrom(in)); directives.add(CacheDirectiveInfoProto.parseDelimitedFrom(in));
fsn.getCacheManager().loadState( fsn.getCacheManager().loadState(