HDFS-6256. Clean up ImageVisitor and SpotCheckImageVisitor. Contributed by Akira Ajisaka.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1588524 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Haohui Mai
parent
bad021534c
commit
bc0b7c6145
|
|
@ -302,6 +302,9 @@ Release 2.5.0 - UNRELEASED
|
||||||
HDFS-6219. Proxy superuser configuration should use true client IP for
|
HDFS-6219. Proxy superuser configuration should use true client IP for
|
||||||
address checks. (daryn via kihwal)
|
address checks. (daryn via kihwal)
|
||||||
|
|
||||||
|
HDFS-6256. Clean up ImageVisitor and SpotCheckImageVisitor.
|
||||||
|
(Akira Ajisaka via wheat9)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
|
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
|
||||||
|
|
|
||||||
|
|
@ -1,212 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An implementation of ImageVisitor can traverse the structure of an
|
|
||||||
* Hadoop fsimage and respond to each of the structures within the file.
|
|
||||||
*/
|
|
||||||
abstract class ImageVisitor {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Structural elements of an FSImage that may be encountered within the
|
|
||||||
* file. ImageVisitors are able to handle processing any of these elements.
|
|
||||||
*/
|
|
||||||
public enum ImageElement {
|
|
||||||
FS_IMAGE,
|
|
||||||
IMAGE_VERSION,
|
|
||||||
NAMESPACE_ID,
|
|
||||||
IS_COMPRESSED,
|
|
||||||
COMPRESS_CODEC,
|
|
||||||
LAYOUT_VERSION,
|
|
||||||
NUM_INODES,
|
|
||||||
GENERATION_STAMP,
|
|
||||||
GENERATION_STAMP_V2,
|
|
||||||
GENERATION_STAMP_V1_LIMIT,
|
|
||||||
LAST_ALLOCATED_BLOCK_ID,
|
|
||||||
INODES,
|
|
||||||
INODE,
|
|
||||||
INODE_PATH,
|
|
||||||
REPLICATION,
|
|
||||||
MODIFICATION_TIME,
|
|
||||||
ACCESS_TIME,
|
|
||||||
BLOCK_SIZE,
|
|
||||||
NUM_BLOCKS,
|
|
||||||
BLOCKS,
|
|
||||||
BLOCK,
|
|
||||||
BLOCK_ID,
|
|
||||||
NUM_BYTES,
|
|
||||||
NS_QUOTA,
|
|
||||||
DS_QUOTA,
|
|
||||||
PERMISSIONS,
|
|
||||||
SYMLINK,
|
|
||||||
NUM_INODES_UNDER_CONSTRUCTION,
|
|
||||||
INODES_UNDER_CONSTRUCTION,
|
|
||||||
INODE_UNDER_CONSTRUCTION,
|
|
||||||
PREFERRED_BLOCK_SIZE,
|
|
||||||
CLIENT_NAME,
|
|
||||||
CLIENT_MACHINE,
|
|
||||||
USER_NAME,
|
|
||||||
GROUP_NAME,
|
|
||||||
PERMISSION_STRING,
|
|
||||||
CURRENT_DELEGATION_KEY_ID,
|
|
||||||
NUM_DELEGATION_KEYS,
|
|
||||||
DELEGATION_KEYS,
|
|
||||||
DELEGATION_KEY,
|
|
||||||
DELEGATION_TOKEN_SEQUENCE_NUMBER,
|
|
||||||
NUM_DELEGATION_TOKENS,
|
|
||||||
DELEGATION_TOKENS,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_KIND,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_SEQNO,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_OWNER,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_RENEWER,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_REALUSER,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_MAX_DATE,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME,
|
|
||||||
DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID,
|
|
||||||
TRANSACTION_ID,
|
|
||||||
LAST_INODE_ID,
|
|
||||||
INODE_ID,
|
|
||||||
|
|
||||||
SNAPSHOT_COUNTER,
|
|
||||||
NUM_SNAPSHOTS_TOTAL,
|
|
||||||
NUM_SNAPSHOTS,
|
|
||||||
SNAPSHOTS,
|
|
||||||
SNAPSHOT,
|
|
||||||
SNAPSHOT_ID,
|
|
||||||
SNAPSHOT_ROOT,
|
|
||||||
SNAPSHOT_QUOTA,
|
|
||||||
NUM_SNAPSHOT_DIR_DIFF,
|
|
||||||
SNAPSHOT_DIR_DIFFS,
|
|
||||||
SNAPSHOT_DIR_DIFF,
|
|
||||||
SNAPSHOT_DIFF_SNAPSHOTID,
|
|
||||||
SNAPSHOT_DIR_DIFF_CHILDREN_SIZE,
|
|
||||||
SNAPSHOT_INODE_FILE_ATTRIBUTES,
|
|
||||||
SNAPSHOT_INODE_DIRECTORY_ATTRIBUTES,
|
|
||||||
SNAPSHOT_DIR_DIFF_CREATEDLIST,
|
|
||||||
SNAPSHOT_DIR_DIFF_CREATEDLIST_SIZE,
|
|
||||||
SNAPSHOT_DIR_DIFF_CREATED_INODE,
|
|
||||||
SNAPSHOT_DIR_DIFF_DELETEDLIST,
|
|
||||||
SNAPSHOT_DIR_DIFF_DELETEDLIST_SIZE,
|
|
||||||
SNAPSHOT_DIR_DIFF_DELETED_INODE,
|
|
||||||
IS_SNAPSHOTTABLE_DIR,
|
|
||||||
IS_WITHSNAPSHOT_DIR,
|
|
||||||
SNAPSHOT_FILE_DIFFS,
|
|
||||||
SNAPSHOT_FILE_DIFF,
|
|
||||||
NUM_SNAPSHOT_FILE_DIFF,
|
|
||||||
SNAPSHOT_FILE_SIZE,
|
|
||||||
SNAPSHOT_DST_SNAPSHOT_ID,
|
|
||||||
SNAPSHOT_LAST_SNAPSHOT_ID,
|
|
||||||
SNAPSHOT_REF_INODE_ID,
|
|
||||||
SNAPSHOT_REF_INODE,
|
|
||||||
|
|
||||||
CACHE_NEXT_ENTRY_ID,
|
|
||||||
CACHE_NUM_POOLS,
|
|
||||||
CACHE_POOL_NAME,
|
|
||||||
CACHE_POOL_OWNER_NAME,
|
|
||||||
CACHE_POOL_GROUP_NAME,
|
|
||||||
CACHE_POOL_PERMISSION_STRING,
|
|
||||||
CACHE_POOL_WEIGHT,
|
|
||||||
CACHE_NUM_ENTRIES,
|
|
||||||
CACHE_ENTRY_PATH,
|
|
||||||
CACHE_ENTRY_REPLICATION,
|
|
||||||
CACHE_ENTRY_POOL_NAME
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Begin visiting the fsimage structure. Opportunity to perform
|
|
||||||
* any initialization necessary for the implementing visitor.
|
|
||||||
*/
|
|
||||||
abstract void start() throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Finish visiting the fsimage structure. Opportunity to perform any
|
|
||||||
* clean up necessary for the implementing visitor.
|
|
||||||
*/
|
|
||||||
abstract void finish() throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Finish visiting the fsimage structure after an error has occurred
|
|
||||||
* during the processing. Opportunity to perform any clean up necessary
|
|
||||||
* for the implementing visitor.
|
|
||||||
*/
|
|
||||||
abstract void finishAbnormally() throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Visit non enclosing element of fsimage with specified value.
|
|
||||||
*
|
|
||||||
* @param element FSImage element
|
|
||||||
* @param value Element's value
|
|
||||||
*/
|
|
||||||
abstract void visit(ImageElement element, String value) throws IOException;
|
|
||||||
|
|
||||||
// Convenience methods to automatically convert numeric value types to strings
|
|
||||||
void visit(ImageElement element, int value) throws IOException {
|
|
||||||
visit(element, Integer.toString(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
void visit(ImageElement element, long value) throws IOException {
|
|
||||||
visit(element, Long.toString(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Begin visiting an element that encloses another element, such as
|
|
||||||
* the beginning of the list of blocks that comprise a file.
|
|
||||||
*
|
|
||||||
* @param element Element being visited
|
|
||||||
*/
|
|
||||||
abstract void visitEnclosingElement(ImageElement element)
|
|
||||||
throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Begin visiting an element that encloses another element, such as
|
|
||||||
* the beginning of the list of blocks that comprise a file.
|
|
||||||
*
|
|
||||||
* Also provide an additional key and value for the element, such as the
|
|
||||||
* number items within the element.
|
|
||||||
*
|
|
||||||
* @param element Element being visited
|
|
||||||
* @param key Key describing the element being visited
|
|
||||||
* @param value Value associated with element being visited
|
|
||||||
*/
|
|
||||||
abstract void visitEnclosingElement(ImageElement element,
|
|
||||||
ImageElement key, String value) throws IOException;
|
|
||||||
|
|
||||||
// Convenience methods to automatically convert value types to strings
|
|
||||||
void visitEnclosingElement(ImageElement element,
|
|
||||||
ImageElement key, int value)
|
|
||||||
throws IOException {
|
|
||||||
visitEnclosingElement(element, key, Integer.toString(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
void visitEnclosingElement(ImageElement element,
|
|
||||||
ImageElement key, long value)
|
|
||||||
throws IOException {
|
|
||||||
visitEnclosingElement(element, key, Long.toString(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Leave current enclosing element. Called, for instance, at the end of
|
|
||||||
* processing the blocks that compromise a file.
|
|
||||||
*/
|
|
||||||
abstract void leaveEnclosingElement() throws IOException;
|
|
||||||
}
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ImageVisitor to spot check an fsimage and generate several statistics
|
|
||||||
* about it that we can compare with known values to give a reasonable
|
|
||||||
* assertion that the image was processed correctly.
|
|
||||||
*/
|
|
||||||
class SpotCheckImageVisitor extends ImageVisitor {
|
|
||||||
|
|
||||||
// Statistics gathered by the visitor for Inodes and InodesUnderConstruction
|
|
||||||
static public class ImageInfo {
|
|
||||||
public long totalNumBlocks = 0; // Total number of blocks in section
|
|
||||||
public Set<String> pathNames = new HashSet<String>(); // All path names
|
|
||||||
public long totalFileSize = 0; // Total size of all the files
|
|
||||||
public long totalReplications = 0; // Sum of all the replications
|
|
||||||
}
|
|
||||||
|
|
||||||
final private ImageInfo inodes = new ImageInfo();
|
|
||||||
final private ImageInfo INUCs = new ImageInfo();
|
|
||||||
private ImageInfo current = null;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
void visit(ImageElement element, String value) throws IOException {
|
|
||||||
if(element == ImageElement.NUM_BYTES)
|
|
||||||
current.totalFileSize += Long.parseLong(value);
|
|
||||||
else if (element == ImageElement.REPLICATION)
|
|
||||||
current.totalReplications += Long.parseLong(value);
|
|
||||||
else if (element == ImageElement.INODE_PATH)
|
|
||||||
current.pathNames.add(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
void visitEnclosingElement(ImageElement element, ImageElement key,
|
|
||||||
String value) throws IOException {
|
|
||||||
switch(element) {
|
|
||||||
case INODES:
|
|
||||||
current = inodes;
|
|
||||||
break;
|
|
||||||
case INODES_UNDER_CONSTRUCTION:
|
|
||||||
current = INUCs;
|
|
||||||
break;
|
|
||||||
case BLOCKS:
|
|
||||||
current.totalNumBlocks += Long.parseLong(value);
|
|
||||||
break;
|
|
||||||
// OK to not have a default, we're skipping most of the values
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public ImageInfo getINodesInfo() { return inodes; }
|
|
||||||
|
|
||||||
public ImageInfo getINUCsInfo() { return INUCs; }
|
|
||||||
|
|
||||||
// Unnecessary visitor methods
|
|
||||||
@Override
|
|
||||||
void finish() throws IOException {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
void finishAbnormally() throws IOException {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
void leaveEnclosingElement() throws IOException {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
void start() throws IOException {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
void visitEnclosingElement(ImageElement element) throws IOException {}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue